diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/manifest.txt b/tierb-fixed-moonep-full/moonep/tierbprobe/manifest.txt new file mode 100644 index 0000000000000000000000000000000000000000..71859eab931bf49614d2488893c85380f119cab9 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/manifest.txt @@ -0,0 +1,19 @@ +timestamp: 2026-08-29T23:34:05+00:00 +host: sp-aiinfra-interactive-aghosh-7jtjl-8xb200-moe-cg-0 +gpu: NVIDIA B200 x8 +driver: 580.95.05 +study_scripts_branch: granularity-moonep +study_scripts_commit: eec9aee +campaign: moonep +backend: moonep +phase: tierbprobe +gbs: 8 +seq: 4096 +configs: G1 G2 G3 G4 +capacity: 1.25 +repeats: 3 +nsys: 1 +megatron_repo: /workspace/Megatron-MoE-Echo +megatron_branch_required: moonep_backend_async_grad_reduce_token_routing_analysis +megatron_branch_actual: moonep_backend_async_grad_reduce_token_routing_analysis +megatron_commit: 22fd6865 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/pip_freeze.txt b/tierb-fixed-moonep-full/moonep/tierbprobe/pip_freeze.txt new file mode 100644 index 0000000000000000000000000000000000000000..5ca396750e15a20912b7276738be470a4ab4fbe5 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/pip_freeze.txt @@ -0,0 +1,317 @@ +absl-py==2.3.0 +aiohappyeyeballs @ file:///rapids/aiohappyeyeballs-2.6.1-py3-none-any.whl#sha256=f349ba8f4b75cb25c99c5c2d84e997e485204d2902a9597802b0371f09331fb8 +aiohttp @ file:///rapids/aiohttp-3.12.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=94f98e0e5a49f89b252e115844f756c04fc8050f38252a32a3dd994ce8121f10 +aiosignal @ file:///rapids/aiosignal-1.3.2-py2.py3-none-any.whl#sha256=45cde58e409a301715980c2b01d0c28bdde3770d8290b5eb2173759d9acb31a5 +annotated-types==0.7.0 +anyio==4.9.0 +apex @ file:///opt/pytorch/apex/dist/apex-0.1-cp312-cp312-linux_x86_64.whl#sha256=a1508e4a386362e9c8f46c92ccada9110e421212b964c694235d0fd5a224b170 +argon2-cffi==25.1.0 +argon2-cffi-bindings==21.2.0 +arrow==1.3.0 +asciitree @ file:///rapids/asciitree-0.3.3-py3-none-any.whl#sha256=3e507ccfaeb417823158091fdb9d26b0205f6a59ba680b479a71db65c73d01d4 +asttokens==3.0.0 +astunparse==1.6.3 +async-lru==2.0.5 +attrs==25.3.0 +audioread==3.0.1 +babel==2.17.0 +beautifulsoup4==4.13.4 +black==25.1.0 +bleach==6.2.0 +blis==0.7.11 +cachetools @ file:///rapids/cachetools-6.0.0-py3-none-any.whl#sha256=82e73ba88f7b30228b5507dce1a1f878498fc669d972aef2dde4f3a3c24f103e +catalogue==2.0.10 +certifi==2025.4.26 +cffi==1.17.1 +charset-normalizer==3.4.2 +click==8.5.0 +cloudpathlib==0.21.1 +cloudpickle @ file:///rapids/cloudpickle-3.1.1-py3-none-any.whl#sha256=c8c5a44295039331ee9dad40ba100a9c7297b6f988e50e87ccdf3765a668350e +cmake==3.31.6 +comm==0.2.2 +confection==0.1.5 +contourpy==1.3.2 +cuda-bindings @ file:///rapids/cuda_bindings-12.9.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ff0e28d1e34758654b9c961e1f55e4786e49aee6a4dbceaf3cc24c46c672df7e +cuda-python @ file:///rapids/cuda_python-12.9.0-py3-none-any.whl#sha256=926acba49b2c0a0374c61b7c98f337c085199cf51cdfe4d6423c4129c20547a7 +cudf @ file:///rapids/cudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b779a8e8c0e8577b9b560895958496da211bfac49cdc4f11a49e8d18cb82e348 +cudf-polars @ file:///rapids/cudf_polars-25.4.0-py3-none-any.whl#sha256=227b6c5147e93ea86282c8f7219118f916769fa2ca0cdd931e97a148aa66b869 +cugraph @ file:///rapids/cugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b837c2dc2f7b235c702e4f8df64479b8fcbb342f848516d5faf0cdc4b5f31f29 +cugraph-service-client @ file:///rapids/cugraph_service_client-25.4.0-py3-none-any.whl#sha256=7e568c3e41154b9752f5173555ffcc282ac52d72343feeb79653b2e2b1232470 +cugraph-service-server @ file:///rapids/cugraph_service_server-25.4.0-py3-none-any.whl#sha256=a2830dcffafcccb16e12ebf85b98f8f247869d38c01ab4dd8664a1e7439d31be +cuml @ file:///rapids/cuml-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1e90a464a18b094e24797f1676688c5d7696c4cf97c934504214a53d5aa09aa5 +cupy-cuda12x @ file:///rapids/cupy_cuda12x-13.3.0-cp312-cp312-linux_x86_64.whl#sha256=6a2dfa0aa1e8189a10d1534276c08f1d2f73b747fd28bfbe7b9f4ef66d9af73f +cuvs @ file:///rapids/cuvs-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b12f6b9f80e053a3b38542c2947d2516e468c02cad54a13d8b1a4a61d5aa7914 +cycler==0.12.1 +cymem==2.0.11 +Cython==3.1.1 +dask @ file:///rapids/dask-2025.2.0-py3-none-any.whl#sha256=f0fdeef6ceb0a06569d456c9e704f220f7f54e80f3a6ea42ab98cea6bc642b6e +dask-cuda @ file:///rapids/dask_cuda-25.4.0-py3-none-any.whl#sha256=442d87da0a95d00ca2a262b64c76e785a32ad905c82000fa2227932f81390e19 +dask-cudf @ file:///rapids/dask_cudf-25.4.0-py3-none-any.whl#sha256=5e7cea0b26daea17da6e0bb457435e24460f76ba740358a3a18e239d6015c95c +datasets==5.0.1 +debugpy==1.8.14 +decorator==5.2.1 +-e git+https://github.com/deepseek-ai/DeepEP.git@34152ae28f80bcc3ee38d7a12cb2ad87cfd4ea72#egg=deep_ep +defusedxml==0.7.1 +dill==0.4.1 +distributed @ file:///rapids/distributed-2025.2.0-py3-none-any.whl#sha256=368462084be47092d0a835f6c64c9b852da64dc88302b76d5f93bd5131228c67 +distributed-ucxx @ file:///rapids/distributed_ucxx-0.43.0-py3-none-any.whl#sha256=94f60d0ccb0c94ba8dbfa1eaf455874d18aaab17e2ae218c38590e9fdc2d721d +dm-tree==0.1.9 +einops==0.8.1 +execnet==2.1.1 +executing==2.2.0 +expecttest==0.3.0 +fasteners @ file:///rapids/fasteners-0.19-py3-none-any.whl#sha256=758819cb5d94cdedf4e836988b74de396ceacb8e2794d21f82d131fd9ee77237 +fastjsonschema==2.21.1 +fastrlock @ file:///rapids/fastrlock-0.8.3-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_28_x86_64.whl#sha256=5f13ec08f1adb1aa916c384b05ecb7dbebb8df9ea81abd045f60941c6283a670 +filelock==3.18.0 +flash_attn @ file:///opt/pytorch/flash_attn-2.7.4.post1-cp312-cp312-linux_x86_64.whl#sha256=a10455d6c56b387785b7413fd0bbcb392b42b3d4f4b88fda5d86cc8a9ce9182a +fonttools==4.58.1 +fqdn==1.5.1 +frozenlist @ file:///rapids/frozenlist-1.6.0-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=4e1be9111cb6756868ac242b3c2bd1f09d9aea09846e4f5c23715e7afb647103 +fsspec==2025.5.1 +gast==0.6.0 +grpcio==1.62.1 +h11==0.16.0 +hf-xet==1.6.0 +httpcore==1.0.9 +httpx==0.28.1 +huggingface_hub==1.29.0 +hypothesis==6.130.8 +idna==3.10 +importlib_metadata==8.7.0 +iniconfig==2.1.0 +intel-openmp==2021.4.0 +ipykernel==6.29.5 +ipython==9.3.0 +ipython_pygments_lexers==1.1.1 +isoduration==20.11.0 +isort==6.0.1 +jedi==0.19.2 +Jinja2==3.1.6 +joblib==1.5.1 +json5==0.12.0 +jsonpointer==3.0.0 +jsonschema==4.24.0 +jsonschema-specifications==2025.4.1 +jupyter-events==0.12.0 +jupyter-lsp==2.2.5 +jupyter_client==8.6.3 +jupyter_core==5.8.1 +jupyter_server==2.16.0 +jupyter_server_terminals==0.5.3 +jupyterlab==4.4.3 +jupyterlab_code_formatter==3.0.2 +jupyterlab_pygments==0.3.0 +jupyterlab_server==2.27.3 +jupyterlab_tensorboard_pro @ file:///builder/jupyterlab_tensorboard_pro-4.0.0-py2.py3-none-any.whl#sha256=6bfcb5ced3358eb72150f4b2ab91312faef2f0ca14d845e8379d8fa52df6902e +jupytext==1.17.2 +kiwisolver==1.4.8 +kvikio @ file:///rapids/kvikio-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=8b40fd8ea9e4a2974c04b50b7fc26b06094ac807f9a233281d0bafbaa71b1d85 +langcodes==3.5.0 +language_data==1.3.0 +lazy_loader==0.4 +libcudf @ file:///rapids/libcudf-25.4.0-py3-none-linux_x86_64.whl#sha256=238320b4958c1737bba2a609673e2539a78216b811b5498a68e00395d9b021a4 +libcugraph @ file:///rapids/libcugraph-25.4.0-py3-none-linux_x86_64.whl#sha256=29085e901b2f7569ff2653bb1355fe6fad20f2fcaa585f5649da9867d8df766d +libcuml @ file:///rapids/libcuml-25.4.0-py3-none-linux_x86_64.whl#sha256=8e2e444c72b0e995e15a48da37ff12fb410a47a2baa69ba86ccc9b1a3db91e8c +libcuvs @ file:///rapids/libcuvs-25.4.0-py3-none-linux_x86_64.whl#sha256=e2f8db61c92b769c590dd752f43439ddd62b2f6592a3c337362b6c1ef220c3b9 +libkvikio @ file:///rapids/libkvikio-25.4.0-py3-none-linux_x86_64.whl#sha256=da8a5713b05fc478bf40529a3e0fb61a2d605bb5a70ca4d461f96a6005dbe8b7 +libraft @ file:///rapids/libraft-25.4.0-py3-none-linux_x86_64.whl#sha256=b921dec005aa4a321dd156776a18376feb47c96a5b67eec125c8a8e19edd1e24 +librmm @ file:///rapids/librmm-25.4.0-py3-none-linux_x86_64.whl#sha256=ff951811bda729d4fa1109ab906225ae1f489a6ec2eeab5d1b674cc34bd4422f +librmm-cu12 @ file:///rapids/librmm_cu12-25.4.0-py3-none-any.whl#sha256=29296ef2b0e90679a0db280e86015be12d6ab9f4e62a181aa085f62dbadd06b3 +librosa==0.11.0 +libucxx @ file:///rapids/libucxx-0.43.0-py3-none-linux_x86_64.whl#sha256=f15849d7ac2fb68f0fd8998838a13321ddcfb0c594f7e6cbc401a3bcc4750566 +lightning-thunder @ file:///opt/pytorch/lightning-thunder +lightning-utilities==0.14.3 +lintrunner==0.12.7 +llvmlite==0.42.0 +locket @ file:///rapids/locket-1.0.0-py2.py3-none-any.whl#sha256=b6c819a722f7b6bd955b80781788e4a66a55628b858d347536b7e81325a3a5e3 +looseversion==1.3.0 +marisa-trie==1.2.1 +Markdown==3.8 +markdown-it-py==3.0.0 +MarkupSafe==3.0.2 +matplotlib==3.10.3 +matplotlib-inline==0.1.7 +mdit-py-plugins==0.4.2 +mdurl==0.1.2 +-e git+ssh://git@github.com/sriram-17-17/my-Megatron-MoE-Echo.git@22fd686582b2696ab8d805464c14eefb97f0ad35#egg=megatron_core +mistune==3.1.3 +mkl==2021.1.1 +mkl-devel==2021.1.1 +mkl-include==2021.1.1 +ml_dtypes==0.6.0 +mock==5.2.0 +-e git+ssh://git@github.com/sriram-17-17/MoonEP.git@d8bc6667fad7177e6e09b637d773fa67f08bae93#egg=moonep +mpmath==1.3.0 +msgpack==1.1.0 +multidict @ file:///rapids/multidict-6.4.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a920f9cf2abdf6e493c519492d892c362007f113c94da4c239ae88429835bad1 +multiprocess==0.70.19 +murmurhash==1.0.13 +mypy_extensions==1.1.0 +nbclient==0.10.2 +nbconvert==7.16.6 +nbformat==5.10.4 +nest-asyncio==1.6.0 +networkx==3.5 +ninja==1.11.1.4 +notebook==7.4.3 +notebook_shim==0.2.4 +numba==0.59.1 +numba-cuda @ file:///rapids/numba_cuda-0.4.0-py3-none-any.whl#sha256=5b95a5f42068771e37142944d44fe844fa87f1b2f89051461a4cb0af91bcc28a +numcodecs @ file:///rapids/numcodecs-0.13.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=da2230484e6102e5fa3cc1a5dd37ca1f92dfbd183d91662074d6f7574e3e8f53 +numpy==1.26.4 +nvdlfw_inspect @ git+https://github.com/NVIDIA/nvidia-dlfw-inspect.git@6b60eb0e675606fb2fbbfcb12667ecbec75eaf87 +nvfuser==0.2.27a0+9bf5aca +nvidia-cuda-cccl-cu12==12.9.27 +nvidia-cudnn-frontend @ file:///opt/pytorch/lightning-thunder/tmp/cudnn_frontend +nvidia-cutlass-dsl==4.4.2 +nvidia-cutlass-dsl-libs-base==4.4.2 +nvidia-dali-cuda120==1.50.0 +nvidia-ml-py @ file:///rapids/nvidia_ml_py-12.575.51-py3-none-any.whl#sha256=eb8641800d98ce40a22f479873f34b482e214a7e80349c63be51c3919845446e +nvidia-modelopt==0.29.0 +nvidia-modelopt-core==0.29.0 +nvidia-nvcomp-cu12==4.2.0.14 +nvidia-nvimgcodec-cu12==0.5.0.13 +nvidia-nvjpeg-cu12==12.4.0.16 +nvidia-nvjpeg2k-cu12==0.8.1.40 +nvidia-nvshmem-cu12==3.7.2 +nvidia-nvtiff-cu12==0.5.0.67 +nvidia-resiliency-ext==0.4.0 +nvtx @ file:///rapids/nvtx-0.2.11-cp312-cp312-linux_x86_64.whl#sha256=73521e37cba42b4db0b3cd68fc5eb814e55c500a61eeed6edcdf6746315ac53e +nx-cugraph @ file:///rapids/nx_cugraph-25.4.0-py3-none-any.whl#sha256=e625444486cc899a77bd48e7da76a2edd95c0006dd95fbf323745feae7f33111 +onnx @ file:///opt/pytorch/pytorch/third_party/onnx +onnx-ir==1.0.0 +onnxscript==0.7.1 +opt_einsum==3.4.0 +optree==0.16.0 +overrides==7.7.0 +packaging==23.2 +pandas @ file:///rapids/pandas-2.2.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=fffb8ae78d8af97f849404f21411c95062db1496aeb3e56f146f0355c9989319 +pandocfilters==1.5.1 +parso==0.8.4 +partd @ file:///rapids/partd-1.4.2-py3-none-any.whl#sha256=978e4ac767ec4ba5b86c6eaa52e5a2a3bc748a2ca839e8cc798f1cc6ce6efb0f +pathspec==0.12.1 +pexpect==4.9.0 +pillow==11.2.1 +platformdirs==4.3.8 +pluggy==1.6.0 +ply @ file:///rapids/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce +polars @ file:///rapids/polars-1.25.2-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f7fcbb4f476784384ccda48757fca4e8c2e2c5a0a3aef3717aaf56aee4e30e09 +polygraphy==0.49.20 +pooch==1.8.2 +preshed==3.0.10 +prometheus_client==0.22.1 +prompt_toolkit==3.0.51 +propcache @ file:///rapids/propcache-0.3.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=563f9d8c03ad645597b8d010ef4e9eab359faeb11a0a2ac9f7b4bc8c28ebef25 +protobuf==4.24.4 +psutil==7.0.0 +ptyprocess==0.7.0 +PuLP==3.2.1 +pure_eval==0.2.3 +pyarrow==25.0.1 +pybind11==2.13.6 +pybind11_global==2.13.6 +pycocotools @ git+https://github.com/nvidia/cocoapi.git@08350a157b41f0755a536441b5abec89ab934c5b#subdirectory=PythonAPI +pycparser==2.22 +pydantic==2.11.5 +pydantic_core==2.33.2 +Pygments==2.19.1 +pylibcudf @ file:///rapids/pylibcudf-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=401b8bfd022d06f84e120779ca7c3d01a28b68762aaa8f7873b10d6b1975cf6a +pylibcugraph @ file:///rapids/pylibcugraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=061c7389a88c815b54fd5d3a3584fb87bf693e5602f5be335260d0ba575ba557 +pylibcugraphops @ file:///rapids/pylibcugraphops-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=a25ef4b22166c80718d5c7e5bddb755e81a2fe340fa5becd0ac36ec3edfb208a +pylibraft @ file:///rapids/pylibraft-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=f2ecd23fe51f9c2caa87bbbe85cb32fcaff4ecfb52ceb9a9991bc50f82782c2e +pylibwholegraph @ file:///rapids/pylibwholegraph-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=b82913d1e38bda76d8c272fbd8cc9072d987209d3d47cc93a464a37c2925aca2 +pynvjitlink @ file:///rapids/pynvjitlink-0.3.0-cp312-cp312-linux_x86_64.whl#sha256=066caf1314730643d3b48c9847bd65789defdd6a7c78993176eb3d8bbe3f4919 +pynvml @ file:///rapids/pynvml-12.0.0-py3-none-any.whl#sha256=fdff84b62a27dbe98e08e1a647eb77342bef1aebe0878bcd15e99a83fcbecb9e +pyparsing==3.2.3 +pytest==8.1.1 +pytest-flakefinder==1.1.0 +pytest-rerunfailures==15.1 +pytest-shard==0.1.2 +pytest-xdist==3.7.0 +python-dateutil==2.9.0.post0 +python-hostlist==2.2.1 +python-json-logger==3.3.0 +pytorch-triton @ file:///tmp/dist/pytorch_triton-3.3.0%2Bgit96316ce52.nvinternal-cp312-cp312-linux_x86_64.whl#sha256=6d6921772b9dfd0243c055e0d054de42a113e92cabcfdd1d3e80a5ad9c08cf7d +pytz @ file:///rapids/pytz-2023.4-py2.py3-none-any.whl#sha256=f90ef520d95e7c46951105338d918664ebfd6f1d995bd7d153127ce90efafa6a +PyYAML==6.0.2 +pyzmq==26.4.0 +raft-dask @ file:///rapids/raft_dask-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=36cf37ba6ce35454c4729c2cb597a360f22780eefb11d514a16a07137da5eb6c +rapids-dask-dependency @ file:///rapids/rapids_dask_dependency-25.4.0a0-py3-none-any.whl#sha256=484faca626e7760112a7d8ad3426f5bdd720b609006d8dca87c2607076a348bd +rapids-logger @ file:///rapids/rapids_logger-0.1.18-py3-none-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl#sha256=3617b2d86291ac6cb957542895f20fc9737ace29773ca90337e6fb04c73bd120 +referencing==0.36.2 +regex==2024.11.6 +requests==2.32.3 +rfc3339-validator==0.1.4 +rfc3986-validator==0.1.1 +rich==14.0.0 +rmm @ file:///rapids/rmm-25.4.0-cp312-cp312-linux_x86_64.whl#sha256=1824bcab3836296666980a997ddcb9c4bca24dfb873012cfa0a7ecea583ca712 +rpds-py==0.25.1 +safetensors==0.5.3 +scikit-learn==1.6.1 +scipy==1.15.3 +Send2Trash==1.8.3 +setuptools==78.1.1 +shellingham==1.5.4 +six==1.16.0 +smart-open==7.1.0 +sniffio==1.3.1 +sortedcontainers==2.4.0 +soundfile==0.13.1 +soupsieve==2.7 +soxr==0.5.0.post1 +spacy==3.7.5 +spacy-legacy==3.0.12 +spacy-loggers==1.0.5 +srsly==2.5.1 +stack-data==0.6.3 +sympy==1.14.0 +tabulate==0.9.0 +tbb==2021.13.1 +tblib @ file:///rapids/tblib-3.1.0-py3-none-any.whl#sha256=670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e +tensorboard==2.16.2 +tensorboard-data-server==0.7.2 +tensorrt @ file:///workspace/TensorRT-10.11.0.33/python/tensorrt-10.11.0.33-cp312-none-linux_x86_64.whl#sha256=c2f18f72758a3d54a71901495c8fda9e7094a8bcdd2a5b6b58967fbf05733b3a +terminado==0.18.1 +thinc==8.2.5 +threadpoolctl==3.6.0 +thriftpy2 @ file:///rapids/thriftpy2-0.5.2-cp312-cp312-linux_x86_64.whl#sha256=c0c7d939bdc5ccf7739154959938445d5fdf72f1d6e12bd563947a36f67e2147 +tinycss2==1.4.0 +toolz @ file:///rapids/toolz-1.0.0-py3-none-any.whl#sha256=292c8f1c4e7516bf9086f8850935c799a874039c8bcf959d47b600e4c44a6236 +torch @ file:///opt/transfer/torch-2.8.0a0%2B5228986c39.nv25.6-29946474-cp312-cp312-linux_x86_64.whl#sha256=7a3c9ff5288d8102a31296b3d62491f3f47c8820a5c9aaefc7dc0fe20e1e8ad5 +torch_tensorrt @ file:///opt/pytorch/torch_tensorrt/dist/torch_tensorrt-2.8.0a0-cp312-cp312-linux_x86_64.whl#sha256=26f4d7a04e7e6e017a5b917b8b98b83720bb2bb0195f48d25f52d567d286bfe1 +torchao @ file:///opt/pytorch/ao +torchprofile==0.0.4 +torchvision @ file:///opt/transfer/torchvision-0.22.0a0%2B95f10a4e-29946474-cp312-cp312-manylinux_2_28_x86_64.whl#sha256=746a3c674cc61680a86f4a14b374b21af1b5c21f629c42765b40c9494f6b234d +tornado==6.5.1 +tqdm==4.67.1 +traitlets==5.14.3 +transformer_engine @ file:///pvcdata/aiinfra/b-aghosh/wheels/transformer_engine-2.10.0.dev0%2B10f6128d-cp312-cp312-linux_x86_64.whl#sha256=35cf5f4b83b2baad864e12e512bb422e0929409ac05610812fa521d4942d0189 +treelite @ file:///rapids/treelite-4.4.1-py3-none-linux_x86_64.whl#sha256=174741724205540a1a5b37da81483aeff55d89548f42b3461cf07acac7ae96eb +typer==0.16.0 +types-dataclasses==0.6.6 +types-python-dateutil==2.9.0.20250516 +typing-inspection==0.4.1 +typing_extensions==4.14.0 +tzdata @ file:///rapids/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 +ucx-py @ file:///rapids/ucx_py-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=4a452fd8d8c983283ada3d49086e3657dc2ee94fbad08efdd708d15ab6e0b73a +ucxx @ file:///rapids/ucxx-0.43.0-cp312-cp312-linux_x86_64.whl#sha256=f1b4e55cb45c66fe88a2d8f9c2b87751165f3cfec21fbdc9c47c4ddff75e8a5f +-e git+https://github.com/Dots-Infra/UltraEP.git@94cab099b44fffa99a82fea99e7c12d89cf65e4f#egg=ultra_ep +uri-template==1.3.0 +urllib3==2.0.7 +wasabi==1.1.3 +wcwidth==0.2.13 +weasel==0.4.1 +webcolors==24.11.1 +webencodings==0.5.1 +websocket-client==1.8.0 +Werkzeug==3.1.3 +wheel==0.45.1 +wrapt==1.17.2 +xdoctest==1.0.2 +xgboost @ file:///rapids/xgboost-2.1.4-py3-none-manylinux_2_39_x86_64.whl#sha256=25eeca27f4048fc491940b3a0b4b5782ee48c33cb3aec059cef71f8da542930b +xxhash==4.0.1 +yarl @ file:///rapids/yarl-1.20.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=bf099e2432131093cc611623e0b0bcc399b8cddd9a91eded8bfb50402ec35018 +zarr @ file:///rapids/zarr-2.18.7-py3-none-any.whl#sha256=ac3dc4033e9ae4e9d7b5e27c97ea3eaf1003cc0a07f010bd83d5134bf8c4b223 +zict @ file:///rapids/zict-3.0.0-py2.py3-none-any.whl#sha256=5796e36bd0e0cc8cf0fbc1ace6a68912611c1dbd74750a3f3026b9b9d6a327ae +zipp==3.22.0 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/provenance.txt b/tierb-fixed-moonep-full/moonep/tierbprobe/provenance.txt index be3693125da638a9b8cebc8ad930f238952d18e8..6f5c868deba097b684a838c7de38f0a573f8c29e 100644 --- a/tierb-fixed-moonep-full/moonep/tierbprobe/provenance.txt +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/provenance.txt @@ -27,3 +27,66 @@ command : run_ladder.sh --phase tierbprobe --gbs 8 --out /workspace/tierb- 7, 120 MHz, 1965 MHz, 1000.00 W, 28 NVLink topology: + GPU0 GPU1 GPU2 GPU3 GPU4 GPU5 GPU6 GPU7 NIC0 NIC1 NIC2 NIC3 NIC4 NIC5 NIC6 NIC7 NIC8 CPU Affinity NUMA Affinity GPU NUMA ID + GPU0 X NV18 NV18 NV18 NV18 NV18 NV18 NV18 PIX NODE SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU1 NV18 X NV18 NV18 NV18 NV18 NV18 NV18 NODE PIX SYS SYS SYS SYS SYS SYS NODE 0-27 0 N/A + GPU2 NV18 NV18 X NV18 NV18 NV18 NV18 NV18 SYS SYS PIX NODE SYS SYS SYS SYS SYS 28-55 1 N/A + GPU3 NV18 NV18 NV18 X NV18 NV18 NV18 NV18 SYS SYS NODE PIX SYS SYS SYS SYS SYS 28-55 1 N/A + GPU4 NV18 NV18 NV18 NV18 X NV18 NV18 NV18 SYS SYS SYS SYS PIX NODE SYS SYS SYS 56-83 2 N/A + GPU5 NV18 NV18 NV18 NV18 NV18 X NV18 NV18 SYS SYS SYS SYS NODE PIX SYS SYS SYS 56-83 2 N/A + GPU6 NV18 NV18 NV18 NV18 NV18 NV18 X NV18 SYS SYS SYS SYS SYS SYS PIX NODE SYS 84-111 3 N/A + GPU7 NV18 NV18 NV18 NV18 NV18 NV18 NV18 X SYS SYS SYS SYS SYS SYS NODE PIX SYS 84-111 3 N/A + NIC0 PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS SYS NODE + NIC1 NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS SYS NODE + NIC2 SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS SYS SYS + NIC3 SYS SYS NODE PIX SYS SYS SYS SYS SYS SYS NODE X SYS SYS SYS SYS SYS + NIC4 SYS SYS SYS SYS PIX NODE SYS SYS SYS SYS SYS SYS X NODE SYS SYS SYS + unavailable + +---- toolchain ----------------------------------------------- + nvcc : release 12.9, V12.9.86 + gcc : gcc (Ubuntu 13.3.0-6ubuntu2~24.04) 13.3.0 + python : 3.12.3 + torch : 2.8.0a0+5228986c39.nv25.06 cuda=12.9 nccl=2.27.3 + torch arch : ['sm_75', 'sm_80', 'sm_86', 'sm_90', 'sm_100', 'sm_120', 'compute_120'] + transformer_engine: 2.10.0.dev0+10f6128d [/usr/local/lib/python3.12/dist-packages/transformer_engine/__init__.py] + deep_ep : 1.2.1+34152ae [/workspace/DeepEP/deep_ep/__init__.py] + ultra_ep : 1.0.0 [/workspace/UltraEP/ultra_ep/__init__.py] + moonep : 0.0.1 [/workspace/MoonEP/moonep/__init__.py] + megatron : 0.14.0rc5 [None] + flash_attn : 2.7.4.post1 [/usr/local/lib/python3.12/dist-packages/flash_attn/__init__.py] + numpy : 1.26.4 [/usr/local/lib/python3.12/dist-packages/numpy/__init__.py] + huggingface_hub: 1.29.0 [/usr/local/lib/python3.12/dist-packages/huggingface_hub/__init__.py] + +---- repositories -------------------------------------------- + Megatron-MoE-Echo moonep_backend_async_grad_reduce_token_routing_analysis 22fd686582b2 + Megatron-UltraEP ultraep-token-drop-analysis eaf829e7ad21 + UltraEP main 94cab099b44f + DeepEP HEAD 34152ae28f80 (+3 untracked: build output) + MoonEP megatron-ddp-symmetric-params d8bc6667fad7 + blackwell_run_scripts granularity-moonep eec9aee807fa + +---- environment --------------------------------------------- + CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0 + CUDA_DRIVER_VERSION=575.57.08 + CUDA_HOME=/usr/local/cuda + CUDA_MODULE_LOADING=LAZY + CUDA_VERSION=12.9.1.010 + HF_REMOTE_ROOT=tierb-fixed-moonep-full + MOONEP_ASSERT_NO_PENDING_REDUCE=1 + MOONEP_ASYNC_GRAD_REDUCE=1 + NCCL_VERSION=2.27.3 + NVSHMEM_VERSION=3.2.5 + PYTORCH_BUILD_NUMBER=0 + PYTORCH_BUILD_VERSION=2.8.0a0+5228986 + PYTORCH_HOME=/opt/pytorch/pytorch + PYTORCH_VERSION=2.8.0a0+5228986 + TORCH_ALLOW_TF32_CUBLAS_OVERRIDE=1 + TORCH_CUDA_ARCH_LIST=7.5 8.0 8.6 9.0 10.0 12.0+PTX + TORCH_NCCL_USE_COMM_NONBLOCKING=0 + +---- shared library resolution ------------------------------- + libnvshmem_host.so: not in ldconfig + libnccl.so: /lib/x86_64-linux-gnu/libnccl.so.2 + lrwxrwxrwx 1 root root 20 Aug 29 16:41 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so -> libnvshmem_host.so.3 + -rw-r--r-- 1 root root 179045744 Aug 29 16:37 /usr/local/lib/python3.12/dist-packages/nvidia/nvshmem/lib/libnvshmem_host.so.3 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..d255a129d4262fe6d01fda9483c092385e574368 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/mem_samples.csv @@ -0,0 +1,224 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 4 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 819 +1, 819 +2, 819 +3, 819 +4, 819 +5, 819 +6, 819 +7, 819 +0, 1537 +1, 1537 +2, 1537 +3, 1537 +4, 1537 +5, 1537 +6, 1537 +7, 1537 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 7221 +1, 7221 +2, 7585 +3, 7329 +4, 7125 +5, 7329 +6, 7245 +7, 7065 +0, 47099 +1, 47099 +2, 47099 +3, 47099 +4, 47099 +5, 47099 +6, 47099 +7, 47099 +0, 56831 +1, 56775 +2, 56805 +3, 56829 +4, 56559 +5, 56825 +6, 56835 +7, 56869 +0, 58623 +1, 58645 +2, 58635 +3, 58633 +4, 58339 +5, 58667 +6, 58669 +7, 58635 +0, 59729 +1, 59729 +2, 59729 +3, 59731 +4, 59443 +5, 59729 +6, 59729 +7, 59729 +0, 59731 +1, 59731 +2, 59731 +3, 59731 +4, 59443 +5, 59731 +6, 59731 +7, 59731 +0, 80465 +1, 80527 +2, 81659 +3, 80567 +4, 81637 +5, 80527 +6, 81657 +7, 81657 +0, 87633 +1, 87675 +2, 87675 +3, 87675 +4, 87635 +5, 87675 +6, 87675 +7, 87675 +0, 87633 +1, 87675 +2, 87675 +3, 87675 +4, 87635 +5, 87675 +6, 87675 +7, 87675 +0, 87909 +1, 87993 +2, 87907 +3, 87949 +4, 87893 +5, 87973 +6, 88017 +7, 88013 +0, 88273 +1, 88287 +2, 88311 +3, 88297 +4, 88243 +5, 88287 +6, 88335 +7, 88299 +0, 97513 +1, 97475 +2, 97491 +3, 97509 +4, 97415 +5, 97563 +6, 97497 +7, 97581 +0, 120091 +1, 120129 +2, 120129 +3, 120129 +4, 120043 +5, 120129 +6, 120129 +7, 120129 +0, 120091 +1, 120129 +2, 120129 +3, 120129 +4, 120043 +5, 120129 +6, 120129 +7, 120129 +0, 120091 +1, 120129 +2, 119867 +3, 120129 +4, 120043 +5, 120129 +6, 120129 +7, 120129 +0, 24037 +1, 22438 +2, 22438 +3, 22438 +4, 22438 +5, 22438 +6, 22758 +7, 119743 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..f94de2d8976f303aa9e84a4598aeb3a9bfa3966a --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..11c113a50aae4d26f26435adf8d3ab59e6d274a3 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/train.log @@ -0,0 +1,4268 @@ +[23:34:05] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[23:34:14] WARN: moonep_backend_async_grad_reduce_token_routing_analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:34:16.313000 213279 torch/distributed/run.py:766] +W0829 23:34:16.313000 213279 torch/distributed/run.py:766] ***************************************** +W0829 23:34:16.313000 213279 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:34:16.313000 213279 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. False + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank 7 is connected to 67 is connected to peer ranks. 7Expected number of connected peer ranks is : peer ranks. 7Expected number of connected peer ranks is : 7 + +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 1[Gloo] Rank 023 is connected to is connected to 4 is connected to is connected to 7[Gloo] Rank 7 is connected to 77 peer ranks. peer ranks. 67 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : is connected to peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 777Expected number of connected peer ranks is : 77 + + peer ranks. 7 + + +Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank3]:[W829 23:34:34.829248322 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:34:34.829312925 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W829 23:34:34.829363726 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:34:34.829396004 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:34:34.829418698 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank6]:[W829 23:34:34.829793279 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:34:34.830943664 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.034 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:34:34.999469287 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.559 seconds +time to initialize megatron (seconds): 7.881 +[after megatron is initialized] datetime: 2026-08-29 23:34:41 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.final_layernorm.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.6.self_attention.linear_proj.weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:34:45 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Build and save the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +INFO:megatron.core.datasets.gpt_dataset:> total number of epochs: 1 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:34:45 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3461.67, 3479.08) + train/valid/test-data-iterators-setup ..........: (124.19, 141.99) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-29 23:34:45 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 544.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.52 MiB + total_size : 548.52 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [5376, 640, 256, 2816, 1280, 5632, 256, 256, 768, 1920, 14208, 4992, 14848, 896, 1664, 3584, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [43, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [127, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [64, 74, 68, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [107, 99, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [29, 17, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [9344, 2688, 128, 24192, 128, 0, 128, 256, 128, 768, 10496, 128, 128, 2432, 128, 1536, 14208, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[69, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [88, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [55, 52, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [63, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [124, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [10, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [58, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [128, 128, 128, 0, 128, 44416, 128, 128, 128, 6912, 128, 256, 128, 128, 128, 128, 13824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[78, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [111, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [62, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [106, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [121, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [57, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [2304, 128, 128, 512, 128, 12416, 128, 1152, 1280, 16768, 640, 29056, 128, 128, 128, 1664, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [118, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [39, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [11, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, 79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [86, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-29 23:35:09] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 23899.0 | throughput per GPU (TFLOP/s/GPU): 7.6 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115993E+01 | loss scale: 1.0 | grad norm: 44.806 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 67175.38232421875 | max allocated: 67175.3828125 | reserved: 68474.0 | max reserved: 68474.0 + [2026-08-29 23:35:09] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 425.2 | throughput per GPU (TFLOP/s/GPU): 425.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117600E+01 | loss scale: 1.0 | grad norm: 44.335 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=90462 payload_rows_remote=75292 dup_rows=40610 dup_rows_remote=27226 dedup_ratio=0.309830 router_tokens_per_expert=[4990, 33, 5, 429, 62, 707, 85, 0, 35, 224, 1098, 67, 1560, 58, 225, 436, 116, 2032, 7, 25, 1009, 12, 31, 819, 1070, 46, 0, 4034, 2092, 2757, 33, 419, 1022, 1302, 146, 18, 124, 397, 5, 15, 542, 6, 697, 3847, 1, 19, 31, 3549, 11, 59, 12, 18, 0, 248, 22, 5, 98, 11, 32, 543, 225, 43, 127, 128, 2815, 11, 2, 2, 1498, 0, 41, 611, 1268, 2, 2371, 318, 510, 3, 319, 234, 8, 20, 1, 14, 17, 10, 86, 617, 3, 95, 254, 20, 26, 35, 326, 14, 302, 390, 11, 806, 38, 3, 45, 33, 157, 81, 672, 4044, 20, 110, 309, 33, 186, 921, 1, 120, 0, 194, 355, 229, 108, 26, 4684, 374, 57, 83, 24, 1750] payload_rows_per_dst=[15170, 13302, 13534, 9900, 10966, 8914, 11318, 7358] dup_rows_per_dst=[13384, 8476, 8440, 1332, 3414, 1276, 2790, 1498] combine_rows_per_src=[15170, 13302, 13534, 9900, 10966, 8914, 11318, 7358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=90990 payload_rows_remote=77914 dup_rows=40082 dup_rows_remote=34986 dedup_ratio=0.305801 router_tokens_per_expert=[3519, 69, 4, 233, 225, 629, 114, 3, 27, 503, 1266, 204, 1284, 109, 323, 217, 110, 1666, 1, 28, 871, 13, 52, 1711, 1411, 79, 10, 4598, 2362, 3648, 109, 627, 513, 1031, 184, 4, 85, 926, 9, 63, 1077, 8, 634, 2845, 4, 73, 20, 3604, 46, 153, 2, 22, 9, 142, 15, 5, 191, 12, 38, 339, 267, 130, 242, 225, 2788, 6, 3, 0, 1210, 0, 30, 474, 935, 23, 1291, 618, 374, 25, 502, 228, 27, 5, 9, 12, 44, 1, 72, 1218, 4, 118, 246, 3, 31, 155, 262, 13, 333, 187, 16, 941, 43, 4, 69, 102, 160, 323, 332, 3371, 2, 305, 489, 103, 376, 963, 1, 126, 0, 244, 549, 145, 31, 27, 4682, 412, 85, 47, 3, 1395] payload_rows_per_dst=[8302, 9720, 9288, 13578, 11968, 11742, 13316, 13076] dup_rows_per_dst=[2118, 3656, 3750, 6590, 5994, 5596, 7282, 5096] combine_rows_per_src=[8302, 9720, 9288, 13578, 11968, 11742, 13316, 13076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93444 payload_rows_remote=82772 dup_rows=37628 dup_rows_remote=32720 dedup_ratio=0.287079 router_tokens_per_expert=[4187, 30, 0, 260, 86, 688, 67, 0, 38, 414, 590, 187, 1495, 62, 209, 302, 47, 2035, 7, 80, 1104, 6, 33, 1429, 1140, 53, 6, 4776, 2366, 3589, 85, 624, 818, 1377, 246, 43, 137, 515, 6, 71, 1083, 12, 880, 3660, 0, 18, 40, 3528, 16, 47, 1, 7, 16, 259, 6, 10, 163, 46, 36, 203, 264, 112, 333, 188, 2544, 8, 0, 1, 1221, 10, 53, 401, 948, 14, 2712, 151, 524, 15, 363, 221, 27, 2, 0, 58, 32, 0, 116, 477, 8, 75, 376, 21, 136, 76, 96, 19, 164, 116, 23, 455, 34, 14, 142, 30, 127, 191, 421, 3283, 9, 162, 356, 73, 187, 915, 2, 85, 1, 198, 412, 92, 105, 61, 4846, 389, 84, 99, 15, 1604] payload_rows_per_dst=[12014, 13036, 10672, 12526, 10312, 11956, 9596, 13332] dup_rows_per_dst=[5216, 8608, 4908, 4004, 2972, 5458, 1604, 4858] combine_rows_per_src=[12014, 13036, 10672, 12526, 10312, 11956, 9596, 13332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=92914 payload_rows_remote=80556 dup_rows=38158 dup_rows_remote=31814 dedup_ratio=0.291122 router_tokens_per_expert=[3809, 66, 1, 129, 144, 361, 60, 0, 15, 410, 1516, 116, 1640, 145, 154, 463, 167, 1657, 29, 12, 982, 23, 18, 606, 637, 37, 1, 4022, 1765, 3410, 112, 264, 929, 1799, 165, 17, 91, 1108, 23, 130, 1215, 2, 540, 2675, 0, 45, 30, 4077, 8, 42, 3, 14, 1, 190, 16, 8, 237, 17, 11, 204, 291, 22, 157, 140, 2480, 3, 4, 1, 1563, 2, 23, 758, 1428, 94, 2129, 427, 416, 71, 591, 76, 22, 17, 0, 22, 31, 1, 148, 1590, 4, 79, 201, 39, 106, 65, 166, 2, 165, 271, 13, 402, 59, 10, 319, 88, 164, 453, 237, 4029, 2, 280, 388, 105, 215, 764, 0, 37, 0, 362, 375, 65, 70, 27, 4771, 323, 84, 48, 4, 1869] payload_rows_per_dst=[11234, 7690, 11274, 12910, 12358, 12326, 11956, 13166] dup_rows_per_dst=[4788, 1400, 5874, 4676, 6344, 6164, 4050, 4862] combine_rows_per_src=[11234, 7690, 11274, 12910, 12358, 12326, 11956, 13166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=92864 payload_rows_remote=79624 dup_rows=38208 dup_rows_remote=31438 dedup_ratio=0.291504 router_tokens_per_expert=[3767, 4, 1, 195, 77, 466, 130, 0, 30, 382, 2144, 194, 1503, 102, 322, 385, 44, 2077, 11, 37, 754, 3, 50, 1381, 932, 50, 2, 4478, 2106, 2717, 37, 626, 956, 1357, 229, 9, 45, 890, 7, 97, 712, 12, 955, 3234, 10, 80, 78, 3341, 14, 79, 0, 21, 1, 286, 22, 10, 48, 9, 23, 199, 375, 7, 263, 393, 2811, 3, 2, 0, 896, 4, 32, 827, 1069, 25, 2336, 279, 440, 3, 403, 151, 12, 11, 0, 28, 45, 0, 107, 1353, 8, 65, 163, 7, 99, 200, 145, 32, 278, 201, 11, 728, 56, 13, 115, 39, 143, 167, 356, 3466, 9, 169, 376, 111, 134, 892, 4, 25, 1, 223, 514, 233, 69, 8, 4528, 490, 48, 60, 11, 1733] payload_rows_per_dst=[9318, 9240, 10824, 13210, 12018, 11980, 13240, 13034] dup_rows_per_dst=[2552, 2768, 5672, 4680, 5134, 5720, 6770, 4912] combine_rows_per_src=[9318, 9240, 10824, 13210, 12018, 11980, 13240, 13034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93590 payload_rows_remote=80162 dup_rows=37482 dup_rows_remote=31834 dedup_ratio=0.285965 router_tokens_per_expert=[4167, 70, 3, 129, 118, 480, 82, 0, 8, 334, 1213, 119, 1755, 284, 100, 429, 57, 1351, 33, 27, 828, 3, 12, 1761, 1429, 34, 3, 4115, 2310, 3510, 49, 493, 541, 1573, 172, 50, 29, 362, 10, 131, 1103, 2, 796, 2249, 94, 31, 17, 3473, 14, 92, 6, 11, 1, 313, 46, 7, 181, 17, 16, 453, 368, 14, 95, 279, 2954, 26, 1, 0, 808, 0, 10, 1087, 1509, 31, 1926, 227, 624, 10, 175, 216, 29, 1, 4, 20, 16, 0, 161, 1399, 3, 55, 187, 7, 53, 144, 378, 8, 500, 234, 13, 319, 100, 4, 48, 25, 415, 192, 407, 3571, 20, 151, 687, 96, 143, 592, 0, 94, 0, 124, 704, 292, 48, 19, 4635, 297, 60, 45, 32, 1748] payload_rows_per_dst=[12674, 9610, 10896, 13428, 11664, 11440, 10824, 13054] dup_rows_per_dst=[5908, 3540, 4834, 5648, 5266, 4934, 2740, 4612] combine_rows_per_src=[12674, 9610, 10896, 13428, 11664, 11440, 10824, 13054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=616792064 dispatch_meta_bytes=410072 combine_payload_bytes=616792064 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=617202136 ingress_bytes=767787008 link_bytes=767787008 token_bytes=1233994200 weight_movement_bytes=150994944 total_nvlink_bytes=1384989144 weight_to_token_ratio=0.122363 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=19968 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.736620003e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=638271488 dispatch_meta_bytes=451600 combine_payload_bytes=638271488 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=638723088 ingress_bytes=789266432 link_bytes=789266432 token_bytes=1276994576 weight_movement_bytes=150994944 total_nvlink_bytes=1427989520 weight_to_token_ratio=0.118242 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=30720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.000900933e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=678068224 dispatch_meta_bytes=461968 combine_payload_bytes=678068224 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=678530192 ingress_bytes=904560640 link_bytes=904560640 token_bytes=1356598416 weight_movement_bytes=226492416 total_nvlink_bytes=1583090832 weight_to_token_ratio=0.166956 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=20992 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.147110218e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=659914752 dispatch_meta_bytes=449480 combine_payload_bytes=659914752 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=660364232 ingress_bytes=810909696 link_bytes=810909696 token_bytes=1320278984 weight_movement_bytes=150994944 total_nvlink_bytes=1471273928 weight_to_token_ratio=0.114366 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=17024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.028347639e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93284 payload_rows_remote=81148 dup_rows=37788 dup_rows_remote=32588 dedup_ratio=0.288300 router_tokens_per_expert=[4020, 22, 1, 94, 175, 593, 204, 0, 1, 530, 1515, 251, 1302, 146, 213, 739, 120, 1907, 4, 7, 1278, 0, 41, 661, 1297, 26, 1, 4147, 2213, 3243, 174, 372, 744, 1064, 153, 2, 41, 1194, 24, 37, 535, 4, 391, 2278, 0, 26, 34, 4177, 8, 116, 4, 11, 4, 90, 5, 3, 202, 11, 13, 191, 263, 7, 257, 317, 2532, 5, 0, 0, 981, 2, 32, 969, 961, 8, 2406, 403, 600, 0, 510, 279, 8, 0, 0, 14, 25, 0, 68, 1513, 10, 49, 73, 1, 19, 216, 210, 7, 123, 367, 35, 415, 145, 2, 68, 6, 223, 148, 400, 4192, 3, 141, 369, 65, 91, 813, 0, 35, 1, 164, 918, 138, 47, 24, 5019, 400, 50, 59, 10, 1661] payload_rows_per_dst=[9072, 9498, 10120, 12918, 12396, 12136, 13830, 13314] dup_rows_per_dst=[2500, 2278, 3442, 4866, 6342, 5200, 7614, 5546] combine_rows_per_src=[9072, 9498, 10120, 12918, 12396, 12136, 13830, 13314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=652279808 dispatch_meta_bytes=444248 combine_payload_bytes=652279808 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=652724056 ingress_bytes=727777280 link_bytes=727777280 token_bytes=1305003864 weight_movement_bytes=75497472 total_nvlink_bytes=1380501336 weight_to_token_ratio=0.057852 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=30080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.229240334e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=656687104 dispatch_meta_bytes=447984 combine_payload_bytes=656687104 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=657135088 ingress_bytes=807682048 link_bytes=807682048 token_bytes=1313822192 weight_movement_bytes=150994944 total_nvlink_bytes=1464817136 weight_to_token_ratio=0.114928 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.038603 gemm_active_groups=19 gemm_peak_group_rows=34176 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.024254527e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=103788 payload_rows_remote=87418 dup_rows=27284 dup_rows_remote=16680 dedup_ratio=0.208160 router_tokens_per_expert=[0, 11, 0, 0, 0, 4, 2, 2, 0, 19, 0, 0, 0, 0, 1, 0, 8012, 0, 0, 0, 0, 0, 2431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4, 0, 0, 0, 7, 7, 0, 0, 3, 0, 2, 1, 95, 0, 15, 0, 2, 0, 0, 32, 0, 1, 0, 7473, 1, 0, 0, 0, 11, 0, 16, 0, 0, 0, 0, 1, 0, 831, 0, 0, 26, 8151, 5, 0, 1, 0, 7778, 0, 8175, 1, 0, 583, 0, 0, 0, 515, 109, 8145, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 4, 6, 0, 0, 0, 0, 0, 0, 0, 1, 4738, 0, 0, 0, 8174, 33, 59, 0, 0, 0] payload_rows_per_dst=[16334, 4862, 14960, 15030, 16266, 3606, 16360, 16370] dup_rows_per_dst=[46, 0, 56, 218, 1478, 56, 14826, 10604] combine_rows_per_src=[16334, 4862, 14960, 15030, 16266, 3606, 16360, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=664764416 dispatch_meta_bytes=454944 combine_payload_bytes=664764416 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=665219360 ingress_bytes=815759360 link_bytes=815759360 token_bytes=1329983776 weight_movement_bytes=150994944 total_nvlink_bytes=1480978720 weight_to_token_ratio=0.113531 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.044118 gemm_active_groups=19 gemm_peak_group_rows=29184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.034497695e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=90856 payload_rows_remote=76162 dup_rows=40216 dup_rows_remote=26660 dedup_ratio=0.306824 router_tokens_per_expert=[3726, 18, 1, 91, 110, 316, 119, 1, 18, 393, 1066, 174, 1555, 91, 337, 395, 66, 2332, 5, 22, 792, 9, 27, 806, 543, 37, 12, 3925, 2767, 3663, 22, 471, 1090, 2402, 146, 0, 147, 1044, 4, 75, 420, 23, 217, 2399, 4, 18, 16, 3430, 60, 82, 3, 4, 9, 381, 3, 9, 166, 16, 25, 184, 249, 10, 222, 55, 2944, 11, 1, 0, 1134, 2, 22, 703, 1026, 12, 1780, 850, 465, 7, 374, 575, 3, 3, 0, 101, 46, 1, 67, 1643, 8, 66, 153, 4, 14, 69, 311, 19, 236, 388, 52, 874, 26, 10, 47, 20, 198, 109, 373, 4066, 11, 179, 440, 42, 62, 974, 0, 53, 0, 125, 476, 399, 113, 4, 4663, 301, 27, 52, 2, 1977] payload_rows_per_dst=[11752, 14694, 11768, 9440, 10468, 11392, 11342, 10000] dup_rows_per_dst=[5070, 13556, 5776, 1366, 3456, 4838, 2800, 3354] combine_rows_per_src=[11752, 14694, 11768, 9440, 10468, 11392, 11342, 10000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118778 payload_rows_remote=102440 dup_rows=12294 dup_rows_remote=12072 dedup_ratio=0.093796 router_tokens_per_expert=[10, 1, 0, 0, 0, 1, 0, 0, 78, 175, 0, 0, 0, 277, 11, 0, 7900, 3, 0, 0, 0, 0, 7104, 0, 2, 0, 0, 0, 5, 0, 32, 0, 0, 0, 15, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3, 29, 111, 1, 0, 716, 0, 27, 249, 0, 1, 0, 180, 0, 1, 0, 1, 0, 2, 5322, 0, 1, 0, 0, 3, 0, 31, 0, 0, 0, 16, 0, 0, 0, 0, 3, 445, 5811, 1, 3, 1, 1, 8128, 0, 8117, 0, 6, 4, 1, 3, 0, 11, 0, 5463, 2, 0, 0, 23, 0, 0, 0, 1, 0, 0, 36, 0, 0, 0, 2, 0, 3, 35, 0, 0, 8134, 0, 1, 0, 6986, 2, 1, 0, 0, 0] payload_rows_per_dst=[12436, 14288, 16338, 16260, 15876, 16282, 11014, 16284] dup_rows_per_dst=[292, 378, 222, 10336, 32, 926, 36, 72] combine_rows_per_src=[12436, 14288, 16338, 16260, 15876, 16282, 11014, 16284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121744 payload_rows_remote=105408 dup_rows=9328 dup_rows_remote=8512 dedup_ratio=0.071167 router_tokens_per_expert=[0, 0, 0, 3, 0, 29, 2, 0, 0, 3, 1, 0, 0, 8, 0, 0, 8168, 0, 0, 1, 6, 0, 6998, 0, 20, 15, 1, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 2, 0, 106, 0, 0, 0, 0, 0, 0, 2, 0, 0, 226, 37, 4, 0, 0, 0, 0, 5, 0, 433, 0, 85, 351, 5495, 0, 0, 0, 0, 0, 0, 1, 2, 1, 0, 5, 0, 0, 399, 0, 0, 8, 7980, 0, 0, 0, 0, 7603, 13, 8191, 0, 3, 27, 0, 229, 0, 2376, 0, 7799, 0, 0, 1, 0, 3, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 66, 5, 1, 593, 0, 0, 0, 8178, 2, 25, 11, 0, 1] payload_rows_per_dst=[16004, 14058, 16384, 12038, 16336, 15820, 15610, 15494] dup_rows_per_dst=[48, 24, 234, 2324, 816, 4698, 0, 1184] combine_rows_per_src=[16004, 14058, 16384, 12038, 16336, 15820, 15610, 15494] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101522 payload_rows_remote=85164 dup_rows=29550 dup_rows_remote=25280 dedup_ratio=0.225449 router_tokens_per_expert=[0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 5, 0, 5, 8067, 16, 0, 0, 0, 0, 8006, 0, 683, 19, 1, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 1, 2, 6, 2, 1, 0, 0, 0, 0, 0, 2, 0, 0, 5, 56, 8, 0, 0, 0, 0, 0, 5, 0, 1, 31, 0, 3777, 9, 0, 0, 0, 0, 1, 28, 1, 0, 0, 0, 0, 1, 15, 1, 0, 7, 7893, 2, 21, 118, 0, 8170, 0, 8179, 0, 10, 26, 8, 45, 2, 4240, 10, 6631, 0, 1, 0, 2, 0, 5, 0, 0, 0, 1, 15, 1, 0, 0, 0, 0, 0, 715, 1, 0, 1457, 0, 0, 1, 7201, 0, 0, 0, 0, 0] payload_rows_per_dst=[16358, 16372, 16366, 7650, 11808, 15318, 13308, 4342] dup_rows_per_dst=[4270, 19910, 24, 116, 10, 5190, 24, 6] combine_rows_per_src=[16358, 16372, 16366, 7650, 11808, 15318, 13308, 4342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110210 payload_rows_remote=95910 dup_rows=20862 dup_rows_remote=15380 dedup_ratio=0.159164 router_tokens_per_expert=[10, 0, 0, 1, 0, 35, 2, 0, 6, 13, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 7, 3173, 2, 0, 0, 0, 0, 0, 6, 0, 14, 0, 7, 0, 0, 0, 8, 0, 1, 13, 0, 0, 0, 0, 63, 0, 1, 0, 0, 0, 0, 173, 0, 0, 0, 1, 0, 1, 0, 0, 0, 4, 6335, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7, 0, 859, 0, 0, 1465, 8113, 0, 0, 0, 0, 8157, 0, 8170, 0, 2, 116, 16, 0, 0, 982, 2, 6931, 0, 1, 0, 0, 0, 11, 0, 0, 1, 1, 200, 23, 0, 15, 0, 2, 0, 256, 0, 311, 3674, 0, 0, 0, 8190, 0, 0, 0, 0, 0] payload_rows_per_dst=[16276, 6396, 16376, 12754, 16316, 11408, 14300, 16384] dup_rows_per_dst=[84, 8, 150, 274, 1726, 4656, 5482, 8482] combine_rows_per_src=[16276, 6396, 16376, 12754, 16316, 11408, 14300, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114850 payload_rows_remote=98466 dup_rows=16222 dup_rows_remote=510 dedup_ratio=0.123764 router_tokens_per_expert=[0, 0, 0, 18, 0, 15, 22, 6, 0, 0, 10, 0, 0, 0, 0, 0, 7779, 0, 0, 1, 0, 0, 7192, 0, 8, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 1, 14, 2, 0, 14, 23, 0, 0, 109, 0, 0, 0, 359, 0, 66, 0, 0, 0, 0, 7801, 0, 0, 0, 14, 0, 0, 0, 3, 0, 0, 24, 19, 0, 3, 0, 0, 50, 8001, 0, 1, 0, 0, 5623, 10, 8161, 0, 1094, 0, 0, 0, 0, 88, 0, 8052, 0, 0, 26, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 1, 2867, 0, 0, 0, 7676, 0, 1, 0, 0, 0] payload_rows_per_dst=[16008, 14386, 16336, 16384, 15618, 13590, 16114, 6414] dup_rows_per_dst=[136, 18, 44, 15712, 66, 142, 44, 60] combine_rows_per_src=[16008, 14386, 16336, 16384, 15618, 13590, 16114, 6414] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=716128256 dispatch_meta_bytes=416392 combine_payload_bytes=716128256 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=716544648 ingress_bytes=942620672 link_bytes=942620672 token_bytes=1432672904 weight_movement_bytes=226492416 total_nvlink_bytes=1659165320 weight_to_token_ratio=0.158091 gemm_rows_defined=67328 gemm_rows_charged=69632 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.033088 gemm_active_groups=18 gemm_peak_group_rows=32256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.195375696e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 2, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=839188480 dispatch_meta_bytes=458048 combine_payload_bytes=839188480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=839646528 ingress_bytes=914685952 link_bytes=914685952 token_bytes=1678835008 weight_movement_bytes=75497472 total_nvlink_bytes=1754332480 weight_to_token_ratio=0.044970 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.159950539e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=863502336 dispatch_meta_bytes=455680 combine_payload_bytes=863502336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=863958016 ingress_bytes=938999808 link_bytes=938999808 token_bytes=1727460352 weight_movement_bytes=75497472 total_nvlink_bytes=1802957824 weight_to_token_ratio=0.043704 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.190783931e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=697663488 dispatch_meta_bytes=441776 combine_payload_bytes=697663488 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=698105264 ingress_bytes=848658432 link_bytes=848658432 token_bytes=1395768752 weight_movement_bytes=150994944 total_nvlink_bytes=1546763696 weight_to_token_ratio=0.108180 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=62080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.076218349e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=785694720 dispatch_meta_bytes=445160 combine_payload_bytes=785694720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=786139880 ingress_bytes=861192192 link_bytes=861192192 token_bytes=1571834600 weight_movement_bytes=75497472 total_nvlink_bytes=1647332072 weight_to_token_ratio=0.048031 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=55040 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.092112921e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=806633472 dispatch_meta_bytes=395904 combine_payload_bytes=806633472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=807029376 ingress_bytes=882130944 link_bytes=882130944 token_bytes=1613662848 weight_movement_bytes=75497472 total_nvlink_bytes=1689160320 weight_to_token_ratio=0.046786 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=46976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.118666206e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=623919104 dispatch_meta_bytes=411288 combine_payload_bytes=623919104 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=624330392 ingress_bytes=925908992 link_bytes=925908992 token_bytes=1248249496 weight_movement_bytes=301989888 total_nvlink_bytes=1550239384 weight_to_token_ratio=0.241931 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=15104 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.174182933e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98474 payload_rows_remote=83192 dup_rows=32598 dup_rows_remote=32178 dedup_ratio=0.248703 router_tokens_per_expert=[0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7, 1, 25, 0, 2234, 0, 1, 0, 0, 0, 0, 0, 9, 2, 0, 0, 31, 0, 0, 0, 0, 0, 0, 7933, 0, 0, 2357, 8162, 0, 0, 0, 6533, 0, 0, 0, 61, 0, 0, 11, 4, 0, 0, 0, 5, 0, 0, 0, 3, 0, 0, 0, 4431, 0, 0, 14, 0, 0, 0, 0, 422, 0, 2, 0, 0, 0, 0, 8021, 0, 0, 11, 9, 0, 0, 7981, 387, 1, 0, 0, 0, 8159, 0, 0, 315, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 22, 0, 18, 0, 73, 0, 434, 107, 0, 4, 0, 0, 0, 0, 0, 19, 2, 0, 0, 0, 7543, 176, 0] payload_rows_per_dst=[4474, 16342, 4824, 8906, 16344, 16358, 15944, 15282] dup_rows_per_dst=[64, 68, 12, 2, 13640, 17368, 1024, 420] combine_rows_per_src=[4474, 16342, 4824, 8906, 16344, 16358, 15944, 15282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108354 payload_rows_remote=96390 dup_rows=22718 dup_rows_remote=22618 dedup_ratio=0.173325 router_tokens_per_expert=[0, 0, 0, 3, 0, 90, 1, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 12, 2, 4, 0, 0, 2, 11, 0, 0, 0, 1, 0, 8182, 0, 0, 0, 7974, 0, 0, 0, 5707, 0, 0, 0, 0, 0, 0, 0, 0, 9, 97, 0, 0, 3, 0, 0, 2, 0, 0, 0, 5687, 0, 0, 0, 0, 0, 1, 0, 747, 0, 0, 3, 0, 0, 0, 7834, 0, 0, 0, 0, 0, 0, 6931, 0, 0, 0, 2, 0, 8088, 0, 5, 3156, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 277, 0, 566, 0, 1414, 0, 0, 153, 26, 0, 0, 0, 260, 0, 0, 0, 0, 0, 71, 0, 8187, 0, 0] payload_rows_per_dst=[13970, 15986, 11964, 15870, 1502, 16302, 16376, 16384] dup_rows_per_dst=[106, 24, 100, 11394, 0, 5552, 4532, 1010] combine_rows_per_src=[13970, 15986, 11964, 15870, 1502, 16302, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113332 payload_rows_remote=113180 dup_rows=17740 dup_rows_remote=17736 dedup_ratio=0.135345 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 494, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 34, 7735, 0, 0, 6900, 8145, 0, 3, 0, 4467, 0, 1, 1, 0, 159, 0, 0, 0, 0, 452, 0, 0, 0, 37, 1, 110, 0, 0, 0, 1798, 0, 0, 3, 0, 0, 9, 0, 62, 4, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 7606, 158, 245, 0, 0, 0, 8058, 0, 0, 0, 0, 0, 3, 0, 0, 0, 99, 1, 1, 2, 0, 0, 0, 1, 0, 1, 14, 2516, 1, 0, 0, 3, 2, 4, 0, 1, 1, 0, 0, 0, 280, 8001, 0, 0] payload_rows_per_dst=[15300, 16296, 16244, 16380, 152, 16278, 16380, 16302] dup_rows_per_dst=[908, 0, 6890, 4644, 4, 644, 4366, 284] combine_rows_per_src=[15300, 16296, 16244, 16380, 152, 16278, 16380, 16302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112622 payload_rows_remote=96242 dup_rows=18450 dup_rows_remote=15826 dedup_ratio=0.140762 router_tokens_per_expert=[0, 0, 0, 0, 0, 27, 0, 216, 1, 2, 2, 0, 0, 0, 0, 0, 7941, 159, 0, 11, 0, 0, 7301, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 6, 0, 1, 54, 0, 0, 0, 82, 0, 0, 2, 0, 0, 0, 7, 0, 10, 0, 0, 0, 0, 4436, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 36, 0, 0, 590, 8041, 0, 0, 1, 0, 8154, 144, 8191, 0, 1, 69, 0, 11, 0, 532, 0, 3757, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 0, 2, 0, 4, 7547, 0, 0, 0, 8184, 0, 0, 0, 0, 0] payload_rows_per_dst=[16280, 14728, 16384, 9064, 15892, 16380, 7526, 16368] dup_rows_per_dst=[298, 216, 126, 10, 70, 2624, 0, 15106] combine_rows_per_src=[16280, 14728, 16384, 9064, 15892, 16380, 7526, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=86810 payload_rows_remote=70446 dup_rows=44262 dup_rows_remote=44232 dedup_ratio=0.337692 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 1, 0, 2181, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 35, 1, 19, 8182, 0, 0, 0, 7673, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 2, 316, 0, 0, 0, 3, 0, 3, 18, 6730, 0, 0, 0, 0, 0, 0, 8131, 9, 0, 0, 0, 0, 2, 8146, 3, 0, 0, 0, 0, 7425, 0, 0, 116, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 5, 117, 0, 17, 2, 0, 0, 1, 0, 0, 0, 0, 1, 1, 8189, 0, 0] payload_rows_per_dst=[4382, 16374, 110, 644, 16372, 16182, 16364, 16382] dup_rows_per_dst=[908, 0, 0, 2, 28762, 14286, 30, 274] combine_rows_per_src=[4382, 16374, 110, 644, 16372, 16182, 16364, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113308 payload_rows_remote=102096 dup_rows=17764 dup_rows_remote=17706 dedup_ratio=0.135529 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 60, 0, 0, 2, 7948, 0, 1, 0, 0, 0, 1, 25, 0, 3, 0, 1, 0, 3, 0, 0, 0, 66, 0, 8135, 183, 0, 53, 8060, 0, 0, 21, 7398, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 42, 27, 0, 0, 0, 0, 0, 0, 0, 4413, 0, 4, 0, 0, 1, 6, 5561, 0, 3, 6, 0, 0, 21, 7676, 32, 0, 0, 0, 0, 3740, 0, 12, 2645, 0, 0, 52, 0, 0, 0, 0, 0, 0, 2, 157, 825, 0, 86, 0, 7, 15, 1, 2, 0, 19, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 8178, 0, 0] payload_rows_per_dst=[16378, 16132, 15272, 11212, 8842, 12738, 16356, 16378] dup_rows_per_dst=[14998, 56, 200, 58, 12, 174, 2204, 62] combine_rows_per_src=[16378, 16132, 15272, 11212, 8842, 12738, 16356, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=681508864 dispatch_meta_bytes=461480 combine_payload_bytes=681508864 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=681970344 ingress_bytes=757006336 link_bytes=757006336 token_bytes=1363479208 weight_movement_bytes=75497472 total_nvlink_bytes=1438976680 weight_to_token_ratio=0.055371 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.599905907e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=103048 payload_rows_remote=86896 dup_rows=28024 dup_rows_remote=19714 dedup_ratio=0.213806 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4945, 0, 0, 0, 0, 0, 2, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8132, 0, 0, 1537, 7853, 0, 0, 0, 7110, 0, 1, 0, 99, 39, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 2149, 1, 0, 1, 0, 0, 0, 0, 8175, 9, 4, 0, 0, 0, 0, 7526, 0, 1, 2, 0, 6, 0, 7275, 416, 12, 0, 0, 0, 483, 0, 0, 1213, 36, 0, 0, 0, 0, 0, 0, 0, 123, 9, 0, 0, 0, 1, 1, 156, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 3, 0, 8007, 0, 12] payload_rows_per_dst=[16152, 16070, 16382, 15522, 16352, 3598, 4058, 14914] dup_rows_per_dst=[8310, 1186, 13386, 3836, 28, 1114, 144, 20] combine_rows_per_src=[16152, 16070, 16382, 15522, 16352, 3598, 4058, 14914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=789626880 dispatch_meta_bytes=476032 combine_payload_bytes=789626880 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=790102912 ingress_bytes=1167114240 link_bytes=1167114240 token_bytes=1579729792 weight_movement_bytes=377487360 total_nvlink_bytes=1957217152 weight_to_token_ratio=0.238957 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.042279 gemm_active_groups=16 gemm_peak_group_rows=33792 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.480065140e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114362 payload_rows_remote=98236 dup_rows=16710 dup_rows_remote=7408 dedup_ratio=0.127487 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 6, 0, 0, 1, 0, 0, 5, 7958, 1, 0, 0, 0, 0, 4659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 2, 3, 305, 0, 3, 0, 63, 0, 0, 0, 0, 18, 34, 9, 1, 15, 0, 0, 0, 0, 6311, 6, 0, 0, 12, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1402, 0, 0, 79, 8088, 0, 0, 1, 2, 8110, 0, 8161, 0, 0, 35, 0, 0, 0, 1961, 0, 8188, 0, 1, 7, 3, 0, 5, 0, 0, 0, 2, 3, 5, 0, 0, 0, 3, 0, 0, 2, 83, 1846, 11, 0, 0, 8054, 0, 0, 0, 6, 0] payload_rows_per_dst=[16196, 16126, 16384, 12832, 16244, 16340, 16380, 3860] dup_rows_per_dst=[6, 9302, 688, 70, 2518, 4036, 48, 42] combine_rows_per_src=[16196, 16126, 16384, 12832, 16244, 16340, 16380, 3860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=927170560 dispatch_meta_bytes=523664 combine_payload_bytes=927170560 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=927694224 ingress_bytes=1078165504 link_bytes=1078165504 token_bytes=1854864784 weight_movement_bytes=150994944 total_nvlink_bytes=2005859728 weight_to_token_ratio=0.081405 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.036765 gemm_active_groups=15 gemm_peak_group_rows=25088 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.367265622e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=788414464 dispatch_meta_bytes=448272 combine_payload_bytes=788414464 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=788862736 ingress_bytes=1165901824 link_bytes=1165901824 token_bytes=1577277200 weight_movement_bytes=377487360 total_nvlink_bytes=1954764560 weight_to_token_ratio=0.239328 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.042279 gemm_active_groups=16 gemm_peak_group_rows=48896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478527626e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=836370432 dispatch_meta_bytes=479208 combine_payload_bytes=836370432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=836849640 ingress_bytes=911867904 link_bytes=911867904 token_bytes=1673220072 weight_movement_bytes=75497472 total_nvlink_bytes=1748717544 weight_to_token_ratio=0.045121 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.034926 gemm_active_groups=17 gemm_peak_group_rows=41472 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.156376857e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=577093632 dispatch_meta_bytes=458712 combine_payload_bytes=577093632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=577552344 ingress_bytes=652591104 link_bytes=652591104 token_bytes=1154645976 weight_movement_bytes=75497472 total_nvlink_bytes=1230143448 weight_to_token_ratio=0.065386 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.040441 gemm_active_groups=16 gemm_peak_group_rows=58240 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.275773790e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=711852032 dispatch_meta_bytes=426440 combine_payload_bytes=711852032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=712278472 ingress_bytes=787349504 link_bytes=787349504 token_bytes=1424130504 weight_movement_bytes=75497472 total_nvlink_bytes=1499627976 weight_to_token_ratio=0.053013 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=44800 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.984699989e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=804749312 dispatch_meta_bytes=422576 combine_payload_bytes=804749312 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=805171888 ingress_bytes=955744256 link_bytes=955744256 token_bytes=1609921200 weight_movement_bytes=150994944 total_nvlink_bytes=1760916144 weight_to_token_ratio=0.093790 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=46976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212018248e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104064 payload_rows_remote=87688 dup_rows=27008 dup_rows_remote=13952 dedup_ratio=0.206055 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7566, 0, 0, 0, 0, 0, 0, 1, 41, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 102, 8184, 0, 0, 0, 0, 0, 11, 0, 7604, 18, 0, 8, 0, 0, 0, 8189, 0, 0, 0, 0, 172, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8117, 24, 0, 0, 0, 0, 0, 0, 24, 0, 1, 0, 8185, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 8121, 0, 0, 0, 190, 0, 1, 0, 8, 0, 0, 0, 0, 0, 2353, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6520, 10, 0, 0, 0, 1, 0] payload_rows_per_dst=[15216, 52, 15264, 8134, 16376, 16262, 16384, 16376] dup_rows_per_dst=[0, 0, 222, 268, 4802, 3560, 5100, 13056] combine_rows_per_src=[15216, 52, 15264, 8134, 16376, 16262, 16384, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112194 payload_rows_remote=100744 dup_rows=18878 dup_rows_remote=18864 dedup_ratio=0.144028 router_tokens_per_expert=[6, 0, 0, 149, 0, 29, 0, 3582, 0, 1501, 0, 0, 0, 0, 680, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 7, 5980, 5769, 0, 3, 25, 0, 0, 6, 0, 8112, 0, 0, 0, 0, 0, 1, 7681, 5, 0, 0, 0, 0, 23, 0, 0, 112, 0, 0, 0, 1, 12, 0, 8180, 0, 0, 0, 0, 0, 5, 0, 5, 0, 1, 8, 5702, 0, 0, 7, 3, 0, 1, 0, 0, 0, 2, 2, 20, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 3122, 0, 0, 0, 226, 0, 0, 0, 0, 0, 4598, 0, 0, 0, 0, 193, 0, 0, 0, 0, 0, 3, 0, 170, 915, 0, 0, 0, 0, 435, 0] payload_rows_per_dst=[10430, 12090, 16262, 16378, 11450, 16384, 16320, 12880] dup_rows_per_dst=[1464, 8, 42, 278, 14, 48, 14934, 2090] combine_rows_per_src=[10430, 12090, 16262, 16378, 11450, 16384, 16320, 12880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109584 payload_rows_remote=109482 dup_rows=21488 dup_rows_remote=21488 dedup_ratio=0.163940 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8167, 7916, 5, 0, 3, 1, 0, 0, 0, 32, 0, 1, 0, 0, 0, 14, 8006, 0, 0, 57, 0, 0, 3, 0, 0, 0, 356, 0, 0, 0, 607, 0, 7586, 0, 0, 0, 24, 0, 0, 0, 0, 0, 161, 0, 8144, 0, 0, 18, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7865, 1, 6, 9, 64, 0, 288, 0, 0, 0, 0, 1, 5, 11, 1924, 0, 1, 0, 0, 1, 0, 5, 0, 16, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[12478, 16348, 102, 16240, 16384, 15744, 16258, 16030] dup_rows_per_dst=[0, 15830, 0, 978, 312, 0, 4360, 8] combine_rows_per_src=[12478, 16348, 102, 16240, 16384, 15744, 16258, 16030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110358 payload_rows_remote=93996 dup_rows=20714 dup_rows_remote=15024 dedup_ratio=0.158035 router_tokens_per_expert=[0, 0, 0, 0, 6, 0, 0, 6433, 0, 185, 0, 0, 0, 0, 3, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7206, 8115, 0, 0, 3, 2, 0, 9, 0, 3356, 7, 0, 1, 0, 1, 0, 3624, 0, 0, 0, 0, 16, 0, 0, 0, 2762, 0, 21, 0, 0, 100, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 58, 0, 0, 0, 6, 0, 0, 0, 0, 0, 748, 0, 0, 0, 0, 1, 0, 0, 422, 0, 0, 0, 0, 0, 7998, 0, 1, 1, 0, 0, 0] payload_rows_per_dst=[13236, 16298, 6734, 16362, 16366, 16278, 8740, 16344] dup_rows_per_dst=[20, 11544, 24, 5690, 0, 0, 132, 3304] combine_rows_per_src=[13236, 16298, 6734, 16362, 16366, 16278, 8740, 16344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116992 payload_rows_remote=100678 dup_rows=14080 dup_rows_remote=13412 dedup_ratio=0.107422 router_tokens_per_expert=[0, 0, 5, 0, 1, 0, 0, 8098, 0, 53, 0, 0, 0, 1, 169, 0, 2, 8, 0, 0, 0, 0, 1, 0, 6, 10, 0, 0, 0, 7918, 8099, 0, 0, 0, 0, 1, 82, 0, 8053, 2, 0, 0, 0, 0, 1, 8003, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1, 0, 1, 0, 1, 4, 8006, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 8165, 16, 0, 0, 40, 0, 0, 0, 0, 2, 0, 16, 0, 0, 429, 0, 0, 0, 1, 0, 0, 0, 125, 0, 0, 0, 0, 0, 42, 0, 2, 0, 0, 34, 0] payload_rows_per_dst=[16228, 3440, 16126, 16032, 16256, 16344, 16314, 16252] dup_rows_per_dst=[426, 52, 12550, 4, 0, 28, 668, 352] combine_rows_per_src=[16228, 3440, 16126, 16032, 16256, 16344, 16314, 16252] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=718340096 dispatch_meta_bytes=406560 combine_payload_bytes=718340096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=718746656 ingress_bytes=793837568 link_bytes=793837568 token_bytes=1437086752 weight_movement_bytes=75497472 total_nvlink_bytes=1512584224 weight_to_token_ratio=0.052535 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.036765 gemm_active_groups=17 gemm_peak_group_rows=38912 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.006697777e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108758 payload_rows_remote=99310 dup_rows=22314 dup_rows_remote=22110 dedup_ratio=0.170242 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 3796, 8184, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 0, 7, 0, 0, 571, 0, 0, 0, 0, 0, 25, 0, 0, 0, 4481, 0, 0, 0, 0, 0, 0, 2, 1217, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 7541, 2, 0, 0, 0, 0, 2397, 0, 0, 2427, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 55, 0, 0, 0, 7, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8171, 1, 0] payload_rows_per_dst=[15234, 16368, 9488, 12636, 12840, 9448, 16384, 16360] dup_rows_per_dst=[4310, 0, 1762, 6136, 9768, 204, 112, 22] combine_rows_per_src=[15234, 16368, 9488, 12636, 12840, 9448, 16384, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108074 payload_rows_remote=91822 dup_rows=22998 dup_rows_remote=12432 dedup_ratio=0.175461 router_tokens_per_expert=[0, 0, 0, 29, 0, 0, 0, 8083, 0, 0, 41, 4, 0, 0, 1, 0, 3, 0, 0, 0, 0, 39, 0, 0, 0, 802, 0, 0, 0, 5251, 5820, 0, 2, 0, 0, 0, 178, 0, 5222, 0, 0, 0, 0, 0, 0, 7974, 0, 0, 0, 0, 59, 0, 0, 0, 75, 810, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8192, 11, 0, 0, 0, 48, 0, 2, 56, 0, 0, 0, 26, 28, 0, 0, 0, 0, 7, 0, 6906, 2, 0, 0, 3012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 89, 0, 3, 0, 1, 0, 1, 0, 0, 0, 0, 2, 7, 30, 4581, 0, 0, 0, 0, 0, 12] payload_rows_per_dst=[16252, 13824, 16382, 15872, 16384, 14004, 6108, 9248] dup_rows_per_dst=[10566, 1612, 10370, 158, 122, 50, 102, 18] combine_rows_per_src=[16252, 13824, 16382, 15872, 16384, 14004, 6108, 9248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=825294848 dispatch_meta_bytes=478432 combine_payload_bytes=825294848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=825773280 ingress_bytes=900792320 link_bytes=900792320 token_bytes=1651068128 weight_movement_bytes=75497472 total_nvlink_bytes=1726565600 weight_to_token_ratio=0.045726 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=62848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.142331458e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=896876544 dispatch_meta_bytes=523880 combine_payload_bytes=896876544 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=897400424 ingress_bytes=1047871488 link_bytes=1047871488 token_bytes=1794276968 weight_movement_bytes=150994944 total_nvlink_bytes=1945271912 weight_to_token_ratio=0.084154 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=46208 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.328848546e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104128 payload_rows_remote=87960 dup_rows=26944 dup_rows_remote=26942 dedup_ratio=0.205566 router_tokens_per_expert=[0, 0, 0, 0, 13, 0, 0, 315, 0, 17, 1, 0, 2, 0, 0, 190, 0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 8134, 0, 1, 1311, 8079, 0, 0, 0, 8044, 11, 0, 0, 0, 0, 0, 9, 0, 0, 839, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2959, 0, 0, 0, 0, 0, 0, 0, 3222, 0, 3, 0, 0, 0, 0, 8107, 2, 0, 0, 3, 0, 0, 7251, 3, 0, 3, 0, 2, 7851, 0, 0, 402, 0, 0, 1, 0, 31, 0, 3, 1, 4, 16, 1, 39, 6, 0, 3, 41, 31, 88, 271, 0, 24, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0] payload_rows_per_dst=[15246, 16168, 16384, 16298, 6454, 826, 16370, 16382] dup_rows_per_dst=[332, 2, 18054, 7532, 0, 0, 428, 596] combine_rows_per_src=[15246, 16168, 16384, 16298, 6454, 826, 16370, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=770015232 dispatch_meta_bytes=436080 combine_payload_bytes=770015232 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=770451312 ingress_bytes=996507648 link_bytes=996507648 token_bytes=1540466544 weight_movement_bytes=226492416 total_nvlink_bytes=1766958960 weight_to_token_ratio=0.147028 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.042279 gemm_active_groups=14 gemm_peak_group_rows=59392 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.263711967e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=824754176 dispatch_meta_bytes=456360 combine_payload_bytes=824754176 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=825210536 ingress_bytes=900251648 link_bytes=900251648 token_bytes=1649964712 weight_movement_bytes=75497472 total_nvlink_bytes=1725462184 weight_to_token_ratio=0.045757 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=48256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.141645810e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=752205824 dispatch_meta_bytes=417016 combine_payload_bytes=752205824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=752622840 ingress_bytes=827703296 link_bytes=827703296 token_bytes=1504828664 weight_movement_bytes=75497472 total_nvlink_bytes=1580326136 weight_to_token_ratio=0.050170 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.042279 gemm_active_groups=14 gemm_peak_group_rows=56192 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.049644287e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=813547520 dispatch_meta_bytes=485680 combine_payload_bytes=813547520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=814033200 ingress_bytes=1040039936 link_bytes=1040039936 token_bytes=1627580720 weight_movement_bytes=226492416 total_nvlink_bytes=1854073136 weight_to_token_ratio=0.139159 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=37632 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.318917036e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=720568320 dispatch_meta_bytes=459608 combine_payload_bytes=720568320 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=721027928 ingress_bytes=871563264 link_bytes=871563264 token_bytes=1441596248 weight_movement_bytes=150994944 total_nvlink_bytes=1592591192 weight_to_token_ratio=0.104741 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.105264901e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=95332 payload_rows_remote=79246 dup_rows=35740 dup_rows_remote=30706 dedup_ratio=0.272675 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 320, 80, 27, 0, 18, 2, 0, 609, 0, 0, 0, 1441, 0, 0, 0, 23, 0, 0, 0, 5645, 14, 0, 0, 8173, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5223, 8111, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 431, 8111, 0, 0, 0, 0, 708, 0, 0, 0, 7978, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 746, 0, 0, 0, 0, 845, 0, 14, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 767, 0, 0, 0, 0, 0, 0, 0, 0, 8036, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[12648, 3892, 2, 13596, 16384, 16350, 16374, 16086] dup_rows_per_dst=[750, 286, 0, 12722, 12290, 3082, 1576, 5034] combine_rows_per_src=[12648, 3892, 2, 13596, 16384, 16350, 16374, 16086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115842 payload_rows_remote=99586 dup_rows=15230 dup_rows_remote=14990 dedup_ratio=0.116196 router_tokens_per_expert=[0, 3, 0, 0, 0, 577, 0, 0, 0, 0, 0, 6857, 0, 0, 0, 0, 0, 4, 1532, 1, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 573, 0, 0, 0, 8188, 8110, 0, 0, 0, 306, 0, 0, 0, 0, 0, 19, 0, 0, 0, 7906, 4031, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 1, 7605, 0, 0, 0, 0, 110, 1, 0, 0, 8113, 4, 0, 0, 14, 0, 6, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 3, 0, 7, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6890, 0, 0, 4462, 0, 0, 0, 0, 0, 0, 102] payload_rows_per_dst=[16372, 3192, 16378, 16194, 16256, 15260, 16224, 15966] dup_rows_per_dst=[6564, 46, 648, 774, 240, 14, 2, 6942] combine_rows_per_src=[16372, 3192, 16378, 16194, 16256, 15260, 16224, 15966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115362 payload_rows_remote=99074 dup_rows=15710 dup_rows_remote=15706 dedup_ratio=0.119858 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 860, 0, 0, 3, 6, 0, 0, 1, 0, 0, 0, 0, 0, 7339, 0, 248, 73, 0, 0, 0, 0, 7922, 0, 0, 0, 8144, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7982, 7189, 0, 0, 0, 23, 32, 0, 0, 0, 0, 0, 0, 7992, 0, 0, 0, 0, 9, 0, 0, 0, 1172, 0, 0, 0, 0, 0, 0, 32, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 154, 1, 0, 0, 0, 0, 0, 0, 0, 0, 47, 24, 0, 0, 0, 13, 0, 0, 25, 7375, 4, 642, 46, 0, 0, 0, 0, 20, 0, 0] payload_rows_per_dst=[15950, 16374, 16288, 16068, 2426, 15988, 16376, 15892] dup_rows_per_dst=[168, 14792, 4, 6, 0, 2, 380, 358] combine_rows_per_src=[15950, 16374, 16288, 16068, 2426, 15988, 16376, 15892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108322 payload_rows_remote=93114 dup_rows=22750 dup_rows_remote=17620 dedup_ratio=0.173569 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4931, 0, 0, 0, 0, 1, 0, 13, 0, 5, 1, 3070, 8, 0, 125, 0, 0, 0, 0, 8074, 0, 0, 0, 8026, 8188, 0, 0, 2, 0, 109, 0, 0, 0, 0, 0, 0, 0, 0, 7536, 1806, 0, 5, 1, 0, 1, 0, 0, 0, 0, 0, 113, 8160, 0, 0, 0, 1, 1, 0, 0, 0, 7959, 0, 0, 0, 0, 0, 3, 0, 2, 1, 0, 71, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 292, 49, 15, 0, 0, 0, 32, 6, 0, 0, 0, 4539, 5, 0, 1, 0, 0, 0, 0, 0, 2041, 0, 0, 269, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[12254, 11424, 16206, 15208, 15930, 16380, 16384, 4536] dup_rows_per_dst=[1220, 6144, 68, 5130, 2, 230, 9870, 86] combine_rows_per_src=[12254, 11424, 16206, 15208, 15930, 16380, 16384, 4536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102286 payload_rows_remote=85902 dup_rows=28786 dup_rows_remote=20430 dedup_ratio=0.219620 router_tokens_per_expert=[0, 0, 0, 6, 0, 0, 0, 0, 0, 14, 1, 863, 0, 0, 0, 0, 1, 4, 1197, 0, 0, 0, 2926, 6, 2, 2, 0, 1, 0, 397, 6403, 0, 0, 0, 8171, 8093, 0, 0, 0, 13, 0, 0, 0, 0, 0, 3, 6, 0, 3, 410, 7039, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 4, 7834, 0, 0, 0, 0, 38, 0, 0, 0, 7716, 0, 102, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 2813, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 206, 0, 0, 0, 0, 3, 12, 1, 0, 0, 4054, 0, 0, 0, 0, 0, 0, 2, 0, 7058, 0, 18, 34, 0, 4, 0, 0, 0, 0, 0] payload_rows_per_dst=[14830, 6318, 5680, 12948, 15658, 16302, 16384, 14166] dup_rows_per_dst=[1016, 2754, 12, 802, 10748, 5032, 8356, 66] combine_rows_per_src=[14830, 6318, 5680, 12948, 15658, 16302, 16384, 14166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=649183232 dispatch_meta_bytes=439808 combine_payload_bytes=649183232 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=649623040 ingress_bytes=800178176 link_bytes=800178176 token_bytes=1298806272 weight_movement_bytes=150994944 total_nvlink_bytes=1449801216 weight_to_token_ratio=0.116257 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.038603 gemm_active_groups=16 gemm_peak_group_rows=49792 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.014738561e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=815808512 dispatch_meta_bytes=458304 combine_payload_bytes=815808512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=816266816 ingress_bytes=891305984 link_bytes=891305984 token_bytes=1632075328 weight_movement_bytes=75497472 total_nvlink_bytes=1707572800 weight_to_token_ratio=0.046259 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.038603 gemm_active_groups=16 gemm_peak_group_rows=52736 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.130301449e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101846 payload_rows_remote=85514 dup_rows=29226 dup_rows_remote=27592 dedup_ratio=0.222977 router_tokens_per_expert=[0, 0, 495, 0, 0, 0, 0, 0, 0, 768, 0, 1, 0, 26, 0, 0, 0, 0, 679, 0, 0, 0, 141, 0, 229, 0, 0, 1, 0, 0, 7296, 0, 0, 0, 7974, 8107, 0, 0, 0, 0, 2, 0, 0, 70, 0, 1, 0, 0, 0, 1414, 7693, 0, 3, 0, 0, 0, 0, 0, 0, 0, 9, 0, 7936, 0, 0, 1, 0, 3, 3, 1, 1, 8153, 0, 15, 32, 0, 0, 1, 0, 0, 7, 0, 0, 0, 1, 0, 1, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 114, 0, 2305, 0, 0, 0, 0, 0, 0, 0, 0, 8089, 17, 8, 3797, 0, 21, 0, 0, 0, 0, 0] payload_rows_per_dst=[16332, 16310, 16338, 8382, 16368, 10390, 4730, 12996] dup_rows_per_dst=[1634, 10504, 15970, 80, 52, 32, 208, 746] combine_rows_per_src=[16332, 16310, 16338, 8382, 16368, 10390, 4730, 12996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126650 payload_rows_remote=110292 dup_rows=4422 dup_rows_remote=4422 dedup_ratio=0.033737 router_tokens_per_expert=[0, 0, 0, 15, 0, 0, 0, 7902, 0, 7, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 7016, 1200, 0, 0, 0, 0, 0, 949, 0, 7762, 18, 0, 0, 0, 3, 0, 7948, 0, 0, 21, 0, 368, 0, 0, 6, 0, 358, 0, 0, 6, 22, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8143, 0, 0, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7261, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15846, 14050, 16124, 16382, 16316, 16358, 15928, 15646] dup_rows_per_dst=[70, 22, 1340, 1542, 2, 0, 164, 1282] combine_rows_per_src=[15846, 14050, 16124, 16382, 16316, 16358, 15928, 15646] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=811614208 dispatch_meta_bytes=459120 combine_payload_bytes=811614208 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=812073328 ingress_bytes=1038106624 link_bytes=1038106624 token_bytes=1623687536 weight_movement_bytes=226492416 total_nvlink_bytes=1850179952 weight_to_token_ratio=0.139493 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=51456 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.316465324e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112492 payload_rows_remote=97326 dup_rows=18580 dup_rows_remote=7844 dedup_ratio=0.141754 router_tokens_per_expert=[0, 0, 10, 0, 0, 0, 0, 8181, 0, 1, 1, 8, 0, 2, 8, 29, 0, 7, 38, 3, 0, 0, 0, 0, 7, 0, 27, 0, 3, 6720, 7310, 16, 0, 0, 1, 0, 241, 2, 6048, 1, 0, 13, 0, 23, 0, 8173, 0, 0, 0, 0, 214, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8103, 0, 10, 0, 0, 0, 21, 0, 0, 0, 0, 0, 1, 0, 13, 0, 0, 1, 60, 0, 7922, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 3, 0, 0, 0, 0, 1, 0, 581, 0, 0, 0, 0, 0, 3508, 0, 0, 2, 1, 0, 0] payload_rows_per_dst=[16380, 15166, 15100, 16360, 16274, 16008, 9560, 7644] dup_rows_per_dst=[2460, 10736, 4430, 394, 14, 4, 0, 542] combine_rows_per_src=[16380, 15166, 15100, 16360, 16274, 16008, 9560, 7644] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=703709184 dispatch_meta_bytes=425328 combine_payload_bytes=703709184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=704134512 ingress_bytes=779206656 link_bytes=779206656 token_bytes=1407843696 weight_movement_bytes=75497472 total_nvlink_bytes=1483341168 weight_to_token_ratio=0.053626 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=51584 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.881437214e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=762789888 dispatch_meta_bytes=442936 combine_payload_bytes=762789888 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=763232824 ingress_bytes=989282304 link_bytes=989282304 token_bytes=1526022712 weight_movement_bytes=226492416 total_nvlink_bytes=1752515128 weight_to_token_ratio=0.148420 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.044118 gemm_active_groups=13 gemm_peak_group_rows=40320 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.254549213e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=700530688 dispatch_meta_bytes=452424 combine_payload_bytes=700530688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=700983112 ingress_bytes=776028160 link_bytes=776028160 token_bytes=1401513800 weight_movement_bytes=75497472 total_nvlink_bytes=1477011272 weight_to_token_ratio=0.053869 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.042279 gemm_active_groups=14 gemm_peak_group_rows=49152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.841129411e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=797294592 dispatch_meta_bytes=420680 combine_payload_bytes=797294592 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=797715272 ingress_bytes=1099284480 link_bytes=1099284480 token_bytes=1595009864 weight_movement_bytes=301989888 total_nvlink_bytes=1896999752 weight_to_token_ratio=0.189334 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.038603 gemm_active_groups=16 gemm_peak_group_rows=35712 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.394047457e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=903512064 dispatch_meta_bytes=458856 combine_payload_bytes=903512064 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=903970920 ingress_bytes=979009536 link_bytes=979009536 token_bytes=1807482984 weight_movement_bytes=75497472 total_nvlink_bytes=1882980456 weight_to_token_ratio=0.041769 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=63488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.241521898e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93436 payload_rows_remote=77366 dup_rows=37636 dup_rows_remote=22332 dedup_ratio=0.287140 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 4449, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 20, 4498, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 1, 0, 0, 10, 0, 1, 0, 3330, 37, 202, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3968, 0, 0, 0, 0, 0, 0, 7927, 3, 0, 151, 0, 193, 3129, 0, 7989, 0, 1, 0, 0, 0, 0, 355, 0, 0, 0, 0, 49, 0, 4991, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7690, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[7698, 634, 16384, 16038, 7528, 13230, 15854, 16070] dup_rows_per_dst=[0, 10, 6850, 8172, 646, 6654, 0, 15304] combine_rows_per_src=[7698, 634, 16384, 16038, 7528, 13230, 15854, 16070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112158 payload_rows_remote=95774 dup_rows=18914 dup_rows_remote=18602 dedup_ratio=0.144302 router_tokens_per_expert=[0, 132, 0, 0, 0, 0, 5487, 0, 0, 8187, 0, 0, 2, 0, 0, 0, 0, 0, 0, 482, 0, 0, 0, 0, 71, 0, 950, 0, 72, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 0, 7953, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 7915, 0, 0, 0, 0, 0, 0, 8190, 0, 7, 0, 0, 0, 8192, 0, 8102, 0, 149, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 134, 180, 59, 0, 2, 0, 0, 8, 0, 0, 0, 343, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[11242, 3124, 16374, 15938, 16384, 16358, 16384, 16354] dup_rows_per_dst=[0, 26, 15940, 58, 312, 1280, 762, 536] combine_rows_per_src=[11242, 3124, 16374, 15938, 16384, 16358, 16384, 16354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115164 payload_rows_remote=98986 dup_rows=15908 dup_rows_remote=14206 dedup_ratio=0.121368 router_tokens_per_expert=[25, 0, 0, 0, 0, 0, 5415, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 14, 0, 63, 8144, 1, 4, 0, 0, 1, 22, 1735, 0, 0, 0, 216, 0, 0, 50, 0, 0, 0, 0, 0, 5, 0, 77, 0, 0, 0, 0, 0, 1419, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 7500, 0, 6, 0, 0, 0, 0, 8080, 0, 0, 0, 17, 0, 2809, 0, 7666, 6, 1308, 0, 0, 0, 0, 4262, 0, 0, 0, 0, 0, 0, 2253, 0, 0, 0, 3, 526, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 5265, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[12040, 16366, 16178, 15700, 14744, 12152, 16296, 11688] dup_rows_per_dst=[358, 4034, 1702, 72, 5042, 1936, 22, 2742] combine_rows_per_src=[12040, 16366, 16178, 15700, 14744, 12152, 16296, 11688] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=107674 payload_rows_remote=91306 dup_rows=23398 dup_rows_remote=23390 dedup_ratio=0.178513 router_tokens_per_expert=[0, 0, 0, 0, 4, 0, 4873, 89, 3, 8155, 0, 0, 0, 11, 2, 0, 0, 2, 0, 4114, 0, 0, 0, 0, 0, 13, 7150, 0, 0, 2, 830, 0, 0, 0, 0, 1, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 334, 0, 1, 0, 0, 0, 0, 0, 0, 45, 0, 0, 4967, 0, 0, 2, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 1268, 0, 7835, 0, 3363, 0, 0, 2, 0, 4, 0, 0, 0, 6, 0, 0, 6529, 0, 0, 0, 0, 0, 0, 0, 3, 2, 0, 0, 0, 0, 21, 0, 0, 0, 2, 0, 0, 0, 1, 7710, 0, 8, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[9944, 14546, 16326, 10604, 8766, 15162, 16368, 15958] dup_rows_per_dst=[20, 2390, 16, 90, 500, 5206, 8, 15168] combine_rows_per_src=[9944, 14546, 16326, 10604, 8766, 15162, 16368, 15958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129266 payload_rows_remote=113884 dup_rows=1806 dup_rows_remote=1798 dedup_ratio=0.013779 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8149, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 544, 0, 0, 0, 0, 3, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 105, 0, 0, 5, 0, 4, 4, 0, 16, 3, 0, 90, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 7591, 0, 0, 0, 0, 0, 0, 8049, 3, 0, 0, 0, 35, 8126, 0, 7901, 0, 107, 0, 0, 0, 0, 39, 0, 15, 0, 1, 0, 0, 8164, 0, 0, 6, 0, 61, 0, 0, 0, 0, 4, 1, 8, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 18, 0, 81, 4, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16298, 16364, 16384, 15382, 16350, 16368, 16122, 15998] dup_rows_per_dst=[0, 1088, 278, 8, 192, 204, 8, 28] combine_rows_per_src=[16298, 16364, 16384, 15382, 16350, 16368, 16122, 15998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100040 payload_rows_remote=83790 dup_rows=31032 dup_rows_remote=21844 dedup_ratio=0.236755 router_tokens_per_expert=[1, 0, 0, 7, 0, 0, 4638, 0, 0, 8072, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6160, 0, 0, 0, 3, 0, 0, 2805, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 8008, 0, 130, 0, 0, 2, 0, 8181, 0, 0, 0, 0, 0, 5283, 0, 8171, 0, 3506, 4, 15, 0, 0, 155, 0, 17, 3, 2, 0, 0, 4283, 0, 0, 0, 0, 0, 463, 1, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 66, 0, 1221, 0, 0, 0, 0, 69, 0, 8, 0, 171, 0, 0, 0, 0, 0] payload_rows_per_dst=[16250, 15126, 7900, 16362, 16376, 8940, 16292, 2794] dup_rows_per_dst=[9188, 8540, 0, 184, 12768, 10, 198, 144] combine_rows_per_src=[16250, 15126, 7900, 16362, 16376, 8940, 16292, 2794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=633782272 dispatch_meta_bytes=398792 combine_payload_bytes=633782272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=634181064 ingress_bytes=709279744 link_bytes=709279744 token_bytes=1267963336 weight_movement_bytes=75497472 total_nvlink_bytes=1343460808 weight_to_token_ratio=0.059542 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.047794 gemm_active_groups=11 gemm_peak_group_rows=41728 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.994665541e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=784580608 dispatch_meta_bytes=457504 combine_payload_bytes=784580608 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=785038112 ingress_bytes=1086570496 link_bytes=1086570496 token_bytes=1569618720 weight_movement_bytes=301989888 total_nvlink_bytes=1871608608 weight_to_token_ratio=0.192397 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=33792 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.377924335e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=810893312 dispatch_meta_bytes=452768 combine_payload_bytes=810893312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=811346080 ingress_bytes=886390784 link_bytes=886390784 token_bytes=1622239392 weight_movement_bytes=75497472 total_nvlink_bytes=1697736864 weight_to_token_ratio=0.046539 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=48384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.124068283e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=747978752 dispatch_meta_bytes=458784 combine_payload_bytes=747978752 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=748437536 ingress_bytes=823476224 link_bytes=823476224 token_bytes=1496416288 weight_movement_bytes=75497472 total_nvlink_bytes=1571913760 weight_to_token_ratio=0.050452 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.036765 gemm_active_groups=17 gemm_peak_group_rows=64512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.044283765e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=932937728 dispatch_meta_bytes=462728 combine_payload_bytes=932937728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933400456 ingress_bytes=1008435200 link_bytes=1008435200 token_bytes=1866338184 weight_movement_bytes=75497472 total_nvlink_bytes=1941835656 weight_to_token_ratio=0.040452 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=54272 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278837782e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109000 payload_rows_remote=92972 dup_rows=22072 dup_rows_remote=16790 dedup_ratio=0.168396 router_tokens_per_expert=[0, 0, 0, 0, 1, 173, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 6959, 0, 0, 0, 1783, 1, 0, 755, 271, 28, 0, 399, 459, 0, 0, 0, 8041, 8126, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 7706, 7091, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7148, 0, 0, 0, 3, 334, 0, 0, 7, 4061, 0, 261, 0, 39, 0, 0, 0, 9, 1, 12, 442, 2, 0, 24, 9, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 39, 297, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 6, 0, 3, 7115, 0, 0, 3693, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14556, 16028, 16254, 15414, 9294, 15224, 5944, 16286] dup_rows_per_dst=[6, 5282, 10840, 0, 134, 136, 320, 5354] combine_rows_per_src=[14556, 16028, 16254, 15414, 9294, 15224, 5944, 16286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122126 payload_rows_remote=105930 dup_rows=8946 dup_rows_remote=8492 dedup_ratio=0.068253 router_tokens_per_expert=[0, 0, 103, 0, 0, 0, 0, 7, 0, 305, 0, 0, 0, 0, 0, 0, 4, 0, 6640, 0, 0, 0, 2214, 0, 3, 2, 0, 0, 0, 0, 4758, 0, 0, 0, 8191, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2107, 7971, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8005, 0, 0, 0, 0, 30, 0, 0, 0, 7553, 0, 60, 0, 2, 0, 0, 0, 0, 2, 0, 0, 6, 0, 0, 0, 312, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 0, 0, 0, 3, 0, 8162, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16348, 13620, 16384, 11784, 15170, 16196, 16288, 16336] dup_rows_per_dst=[424, 4106, 0, 1964, 120, 454, 1656, 222] combine_rows_per_src=[16348, 13620, 16384, 11784, 15170, 16196, 16288, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=686407680 dispatch_meta_bytes=422536 combine_payload_bytes=686407680 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=686830216 ingress_bytes=837402624 link_bytes=837402624 token_bytes=1373237896 weight_movement_bytes=150994944 total_nvlink_bytes=1524232840 weight_to_token_ratio=0.109955 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.036765 gemm_active_groups=15 gemm_peak_group_rows=48384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.061944401e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=867778560 dispatch_meta_bytes=457688 combine_payload_bytes=867778560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=868236248 ingress_bytes=943276032 link_bytes=943276032 token_bytes=1736014808 weight_movement_bytes=75497472 total_nvlink_bytes=1811512280 weight_to_token_ratio=0.043489 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.038603 gemm_active_groups=15 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.196206785e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=761626624 dispatch_meta_bytes=439048 combine_payload_bytes=761626624 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=762065672 ingress_bytes=912621568 link_bytes=912621568 token_bytes=1523692296 weight_movement_bytes=150994944 total_nvlink_bytes=1674687240 weight_to_token_ratio=0.099098 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=21248 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.157332609e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=90812 payload_rows_remote=74654 dup_rows=40260 dup_rows_remote=16630 dedup_ratio=0.307159 router_tokens_per_expert=[6372, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 6, 0, 0, 0, 0, 0, 0, 0, 7296, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7822, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7309, 0, 0, 244, 0, 0, 0, 0, 0, 8192, 141, 0, 552, 0, 0, 0, 0, 0, 0, 688, 0, 0, 0, 8187, 8037, 0, 519, 0, 0, 0, 0, 7831, 1577, 0, 0, 2, 0] payload_rows_per_dst=[12910, 44, 14592, 844, 16374, 16376, 13514, 16158] dup_rows_per_dst=[470, 0, 0, 4, 2, 14378, 1776, 23630] combine_rows_per_src=[12910, 44, 14592, 844, 16374, 16376, 13514, 16158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112400 payload_rows_remote=96106 dup_rows=18672 dup_rows_remote=18670 dedup_ratio=0.142456 router_tokens_per_expert=[294, 2447, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 262, 0, 612, 0, 0, 0, 63, 0, 0, 0, 34, 0, 1, 0, 0, 7972, 0, 97, 12, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 7897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3983, 0, 0, 11, 0, 11, 0, 0, 7, 0, 0, 8065, 0, 0, 0, 0, 2406, 0, 0, 0, 8126, 11, 47, 0, 0, 0, 114, 0, 0, 0, 14, 0, 0, 0, 8146, 8012, 0, 0, 1370, 0, 3, 0, 5230, 0, 0, 0, 0, 0] payload_rows_per_dst=[12958, 16384, 16144, 16348, 16294, 16176, 5088, 13008] dup_rows_per_dst=[1056, 1810, 26, 15472, 2, 12, 96, 198] combine_rows_per_src=[12958, 16384, 16144, 16348, 16294, 16176, 5088, 13008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100408 payload_rows_remote=97708 dup_rows=30664 dup_rows_remote=30662 dedup_ratio=0.233948 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 28, 12, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4092, 0, 66, 0, 7, 0, 122, 0, 90, 0, 0, 1, 0, 0, 0, 0, 789, 0, 0, 42, 1, 29, 0, 0, 0, 0, 0, 0, 1, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6851, 12, 0, 0, 3, 0, 0, 0, 0, 4, 0, 8151, 0, 0, 1446, 0, 490, 0, 0, 0, 8154, 0, 0, 131, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 7738, 1744, 0, 53, 747, 161, 0, 0, 8105, 1, 0, 4, 0, 0] payload_rows_per_dst=[16384, 16384, 2700, 16130, 15484, 16354, 668, 16304] dup_rows_per_dst=[13784, 8680, 2, 3480, 2, 2878, 0, 1838] combine_rows_per_src=[16384, 16384, 2700, 16130, 15484, 16354, 668, 16304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102820 payload_rows_remote=87754 dup_rows=28252 dup_rows_remote=27878 dedup_ratio=0.215546 router_tokens_per_expert=[801, 378, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 5573, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 153, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 186, 0, 1, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 6, 8042, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 1, 530, 3034, 4, 6950, 2, 16, 108, 8192, 42, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1750, 7494, 0, 0, 0, 43, 0, 0, 3317, 0, 0, 0, 0, 0] payload_rows_per_dst=[16180, 16384, 16382, 15066, 3518, 16160, 12412, 6718] dup_rows_per_dst=[2362, 11472, 5744, 374, 0, 6696, 1602, 2] combine_rows_per_src=[16180, 16384, 16382, 15066, 3518, 16160, 12412, 6718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=91434 payload_rows_remote=75050 dup_rows=39638 dup_rows_remote=19280 dedup_ratio=0.302414 router_tokens_per_expert=[5021, 489, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 31, 1, 0, 0, 0, 0, 0, 0, 36, 8128, 0, 0, 0, 0, 0, 49, 5, 0, 42, 0, 0, 0, 0, 217, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5398, 97, 0, 0, 1, 6031, 0, 23, 0, 7240, 4961, 0, 115, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8068, 7924, 0, 0, 0, 0, 0, 2, 2219, 2872, 0, 0, 0, 0] payload_rows_per_dst=[10788, 82, 16372, 492, 16160, 14862, 16384, 16294] dup_rows_per_dst=[242, 0, 148, 62, 4, 9084, 20358, 9740] combine_rows_per_src=[10788, 82, 16372, 492, 16160, 14862, 16384, 16294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97008 payload_rows_remote=82574 dup_rows=34064 dup_rows_remote=33490 dedup_ratio=0.259888 router_tokens_per_expert=[0, 303, 0, 0, 0, 0, 2, 0, 34, 76, 1, 7, 0, 0, 0, 0, 1, 0, 0, 0, 104, 481, 0, 2, 0, 0, 201, 0, 0, 0, 0, 0, 0, 0, 0, 215, 4936, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 789, 0, 7207, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 126, 0, 4, 0, 56, 0, 0, 7081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2424, 0, 35, 122, 0, 36, 0, 28, 0, 7654, 82, 0, 7174, 0, 1, 0, 2, 0, 0, 1363, 0, 0, 0, 8110, 7789, 0, 0, 612, 46, 0, 0, 8099, 13, 0, 0, 311, 0] payload_rows_per_dst=[14434, 16244, 10288, 16378, 16384, 5160, 14600, 3520] dup_rows_per_dst=[574, 642, 94, 15192, 14146, 2, 2700, 714] combine_rows_per_src=[14434, 16244, 10288, 16378, 16384, 5160, 14600, 3520] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=611565568 dispatch_meta_bytes=365136 combine_payload_bytes=611565568 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=611930704 ingress_bytes=913555456 link_bytes=913555456 token_bytes=1223496272 weight_movement_bytes=301989888 total_nvlink_bytes=1525486160 weight_to_token_ratio=0.246825 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.045956 gemm_active_groups=12 gemm_peak_group_rows=39296 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.158516911e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=787300352 dispatch_meta_bytes=459104 combine_payload_bytes=787300352 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=787759456 ingress_bytes=938295296 link_bytes=938295296 token_bytes=1575059808 weight_movement_bytes=150994944 total_nvlink_bytes=1726054752 weight_to_token_ratio=0.095866 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=58368 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.189890510e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=800423936 dispatch_meta_bytes=513480 combine_payload_bytes=800423936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=800937416 ingress_bytes=875921408 link_bytes=875921408 token_bytes=1601361352 weight_movement_bytes=75497472 total_nvlink_bytes=1676858824 weight_to_token_ratio=0.047146 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.034926 gemm_active_groups=17 gemm_peak_group_rows=53120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.110791641e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=718880768 dispatch_meta_bytes=462528 combine_payload_bytes=718880768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=719343296 ingress_bytes=794378240 link_bytes=794378240 token_bytes=1438224064 weight_movement_bytes=75497472 total_nvlink_bytes=1513721536 weight_to_token_ratio=0.052494 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=40704 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.007383426e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98448 payload_rows_remote=82080 dup_rows=32624 dup_rows_remote=18184 dedup_ratio=0.248901 router_tokens_per_expert=[0, 0, 3, 0, 0, 0, 7704, 0, 0, 8115, 0, 9, 0, 0, 4, 0, 163, 0, 0, 6688, 0, 0, 0, 0, 0, 72, 7992, 0, 27, 13, 449, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 155, 0, 1982, 0, 0, 0, 13, 0, 7, 0, 0, 0, 1, 1, 6036, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 451, 0, 7733, 0, 0, 37, 0, 0, 0, 535, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 34, 13, 1, 1045, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16256, 16368, 314, 15346, 15540, 16128, 16368, 2128] dup_rows_per_dst=[15414, 14440, 0, 734, 902, 1070, 6, 58] combine_rows_per_src=[16256, 16368, 314, 15346, 15540, 16128, 16368, 2128] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=614809600 dispatch_meta_bytes=377320 combine_payload_bytes=614809600 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=615186920 ingress_bytes=916799488 link_bytes=916799488 token_bytes=1229996520 weight_movement_bytes=301989888 total_nvlink_bytes=1531986408 weight_to_token_ratio=0.245521 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.044118 gemm_active_groups=15 gemm_peak_group_rows=17024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.162630800e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129194 payload_rows_remote=112838 dup_rows=1878 dup_rows_remote=1860 dedup_ratio=0.014328 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8182, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 500, 0, 0, 0, 319, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 20, 0, 13, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 5, 7297, 0, 7948, 0, 209, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16378, 16380, 16376, 15012, 16356, 16376, 15952] dup_rows_per_dst=[0, 1638, 46, 8, 10, 18, 8, 150] combine_rows_per_src=[16364, 16378, 16380, 16376, 15012, 16356, 16376, 15952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=676446208 dispatch_meta_bytes=464256 combine_payload_bytes=676446208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=676910464 ingress_bytes=751943680 link_bytes=751943680 token_bytes=1353356672 weight_movement_bytes=75497472 total_nvlink_bytes=1428854144 weight_to_token_ratio=0.055785 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=35328 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.535704303e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=672399360 dispatch_meta_bytes=401056 combine_payload_bytes=672399360 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=672800416 ingress_bytes=898891776 link_bytes=898891776 token_bytes=1345199776 weight_movement_bytes=226492416 total_nvlink_bytes=1571692192 weight_to_token_ratio=0.168371 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.040441 gemm_active_groups=16 gemm_peak_group_rows=30720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.139921301e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=924368896 dispatch_meta_bytes=458792 combine_payload_bytes=924368896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=924827688 ingress_bytes=999866368 link_bytes=999866368 token_bytes=1849196584 weight_movement_bytes=75497472 total_nvlink_bytes=1924694056 weight_to_token_ratio=0.040827 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.042279 gemm_active_groups=13 gemm_peak_group_rows=50688 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.267971296e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=106160 payload_rows_remote=91472 dup_rows=24912 dup_rows_remote=21370 dedup_ratio=0.190063 router_tokens_per_expert=[0, 7893, 0, 0, 6311, 0, 0, 0, 0, 0, 0, 8040, 7988, 118, 0, 0, 0, 0, 0, 0, 0, 7570, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 210, 0, 44, 118, 0, 21, 0, 0, 0, 0, 669, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1824, 0, 8189, 655, 5, 0, 6557, 0, 5, 24, 912, 0, 0, 0, 0, 0, 27, 0, 0, 0, 10, 3, 0, 0, 0, 7648, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 230, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 459] payload_rows_per_dst=[236, 16140, 13992, 16384, 14688, 15322, 16244, 13154] dup_rows_per_dst=[0, 14982, 1044, 4952, 3542, 0, 2, 390] combine_rows_per_src=[236, 16140, 13992, 16384, 14688, 15322, 16244, 13154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=91232 payload_rows_remote=74856 dup_rows=39840 dup_rows_remote=24744 dedup_ratio=0.303955 router_tokens_per_expert=[1, 8188, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8169, 8140, 0, 0, 0, 2, 0, 0, 0, 0, 1037, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2245, 39, 6, 0, 0, 0, 0, 326, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7914, 4, 4, 0, 6524, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7127, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 7540] payload_rows_per_dst=[2, 16332, 4492, 6984, 16372, 14290, 16384, 16376] dup_rows_per_dst=[0, 2078, 100, 160, 13052, 9340, 14, 15096] combine_rows_per_src=[2, 16332, 4492, 6984, 16372, 14290, 16384, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113468 payload_rows_remote=97266 dup_rows=17604 dup_rows_remote=2272 dedup_ratio=0.134308 router_tokens_per_expert=[0, 8158, 0, 0, 8175, 0, 0, 0, 255, 0, 5, 7973, 8184, 201, 0, 0, 0, 0, 6, 0, 0, 0, 14, 0, 0, 4, 0, 0, 0, 0, 0, 0, 118, 0, 0, 0, 0, 0, 0, 6, 0, 6, 0, 7659, 5, 0, 0, 0, 0, 0, 0, 20, 0, 16, 0, 61, 0, 0, 0, 0, 7, 0, 8184, 5, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6437, 0, 1, 0, 0, 1643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 7689] payload_rows_per_dst=[16384, 16384, 16202, 16384, 24, 16160, 16380, 15550] dup_rows_per_dst=[888, 32, 15332, 202, 0, 2, 1148, 0] combine_rows_per_src=[16384, 16384, 16202, 16384, 24, 16160, 16380, 15550] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99948 payload_rows_remote=83564 dup_rows=31124 dup_rows_remote=30664 dedup_ratio=0.237457 router_tokens_per_expert=[0, 8180, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8152, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 1101, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 21, 0, 0, 8182, 545, 0, 0, 7854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7852, 0, 0, 36, 0, 5, 5588, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 187, 0, 0, 3, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 17, 20, 0, 0, 0, 0, 0, 58, 1289] payload_rows_per_dst=[0, 16332, 2228, 16384, 16326, 15910, 16384, 16384] dup_rows_per_dst=[0, 44, 0, 1124, 15686, 11056, 460, 2754] combine_rows_per_src=[0, 16332, 2228, 16384, 16326, 15910, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=107372 payload_rows_remote=90988 dup_rows=23700 dup_rows_remote=19024 dedup_ratio=0.180817 router_tokens_per_expert=[0, 8166, 0, 0, 7412, 0, 0, 0, 21, 0, 0, 8192, 8177, 0, 0, 0, 171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 10, 0, 0, 405, 0, 0, 0, 5984, 0, 0, 0, 2, 0, 0, 327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 212, 0, 7776, 2215, 6, 0, 234, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 224, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 1457, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6164] payload_rows_per_dst=[12676, 16354, 16384, 16384, 480, 16384, 16374, 12336] dup_rows_per_dst=[0, 342, 12804, 4676, 2, 3362, 326, 2188] combine_rows_per_src=[12676, 16354, 16384, 16384, 480, 16384, 16374, 12336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=86898 payload_rows_remote=70546 dup_rows=44174 dup_rows_remote=792 dedup_ratio=0.337021 router_tokens_per_expert=[0, 8110, 0, 0, 7948, 0, 9, 0, 0, 0, 0, 4716, 8049, 3, 2, 108, 3230, 0, 0, 0, 0, 35, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 108, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 1, 1, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 34, 0, 10, 0, 8109, 0, 0, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 8, 8130, 0, 664, 43, 0, 5, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 23, 1, 0, 4, 0, 0, 0, 162, 172, 158, 0, 3, 0, 0, 0, 0, 0, 75, 0, 0] payload_rows_per_dst=[16352, 6516, 16220, 16290, 15056, 1392, 14602, 470] dup_rows_per_dst=[43382, 16, 4, 30, 0, 68, 4, 670] combine_rows_per_src=[16352, 6516, 16220, 16290, 15056, 1392, 14602, 470] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=749338624 dispatch_meta_bytes=451368 combine_payload_bytes=749338624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=749789992 ingress_bytes=824836096 link_bytes=824836096 token_bytes=1499128616 weight_movement_bytes=75497472 total_nvlink_bytes=1574626088 weight_to_token_ratio=0.050361 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=38400 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.046008274e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=613220352 dispatch_meta_bytes=398400 combine_payload_bytes=613220352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=613618752 ingress_bytes=688717824 link_bytes=688717824 token_bytes=1226839104 weight_movement_bytes=75497472 total_nvlink_bytes=1302336576 weight_to_token_ratio=0.061538 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=32128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.733911452e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=796803072 dispatch_meta_bytes=398152 combine_payload_bytes=796803072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=797201224 ingress_bytes=872300544 link_bytes=872300544 token_bytes=1594004296 weight_movement_bytes=75497472 total_nvlink_bytes=1669501768 weight_to_token_ratio=0.047363 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.042279 gemm_active_groups=16 gemm_peak_group_rows=35968 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.106199875e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=684556288 dispatch_meta_bytes=456912 combine_payload_bytes=684556288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=685013200 ingress_bytes=760053760 link_bytes=760053760 token_bytes=1369569488 weight_movement_bytes=75497472 total_nvlink_bytes=1445066960 weight_to_token_ratio=0.055125 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.047794 gemm_active_groups=11 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.638551533e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=745373696 dispatch_meta_bytes=440048 combine_payload_bytes=745373696 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=745813744 ingress_bytes=820871168 link_bytes=820871168 token_bytes=1491187440 weight_movement_bytes=75497472 total_nvlink_bytes=1566684912 weight_to_token_ratio=0.050629 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=58624 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.040980187e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102854 payload_rows_remote=86472 dup_rows=28218 dup_rows_remote=24754 dedup_ratio=0.215286 router_tokens_per_expert=[5023, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 18, 22, 21, 0, 0, 0, 0, 0, 0, 1499, 0, 0, 0, 0, 0, 122, 0, 0, 0, 0, 237, 0, 0, 15, 584, 7554, 1, 0, 1, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 17, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 7, 0, 3, 0, 0, 0, 0, 4120, 0, 37, 0, 0, 0, 0, 3, 0, 5, 0, 1367, 124, 0, 0, 0, 7750, 0, 6, 0, 8065, 4820, 0, 3, 0, 0, 0, 0, 0, 0, 4268, 3, 0, 0, 8147, 8152, 0, 92, 0, 1, 0, 0, 3405, 0, 0, 0, 0, 0] payload_rows_per_dst=[13876, 16382, 16356, 16346, 16310, 3032, 13556, 6996] dup_rows_per_dst=[4534, 3464, 15476, 40, 20, 40, 4644, 0] combine_rows_per_src=[13876, 16382, 16356, 16346, 16310, 3032, 13556, 6996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113326 payload_rows_remote=104796 dup_rows=17746 dup_rows_remote=16414 dedup_ratio=0.135391 router_tokens_per_expert=[50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 7822, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 208, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5285, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3053, 0, 741, 16, 0, 4319, 0, 0, 0, 8192, 53, 0, 1339, 0, 0, 0, 0, 0, 0, 2611, 0, 0, 0, 8186, 7449, 0, 0, 10, 0, 0, 0, 7937, 0, 0, 0, 0, 0] payload_rows_per_dst=[10448, 16294, 16372, 14842, 16372, 8530, 14580, 15888] dup_rows_per_dst=[72, 6806, 376, 2, 0, 1332, 9094, 64] combine_rows_per_src=[10448, 16294, 16372, 14842, 16372, 8530, 14580, 15888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=577912832 dispatch_meta_bytes=285352 combine_payload_bytes=577912832 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=578198184 ingress_bytes=1030897664 link_bytes=1030897664 token_bytes=1156111016 weight_movement_bytes=452984832 total_nvlink_bytes=1609095848 weight_to_token_ratio=0.391818 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.044118 gemm_active_groups=14 gemm_peak_group_rows=30592 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.307323348e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=708378624 dispatch_meta_bytes=444904 combine_payload_bytes=708378624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=708823528 ingress_bytes=783876096 link_bytes=783876096 token_bytes=1417202152 weight_movement_bytes=75497472 total_nvlink_bytes=1492699624 weight_to_token_ratio=0.053272 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.044118 gemm_active_groups=14 gemm_peak_group_rows=43904 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.940652286e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=858488832 dispatch_meta_bytes=484840 combine_payload_bytes=858488832 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=858973672 ingress_bytes=1009483776 link_bytes=1009483776 token_bytes=1717462504 weight_movement_bytes=150994944 total_nvlink_bytes=1868457448 weight_to_token_ratio=0.087917 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.038603 gemm_active_groups=16 gemm_peak_group_rows=43648 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.280167524e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110202 payload_rows_remote=93820 dup_rows=20870 dup_rows_remote=18194 dedup_ratio=0.159225 router_tokens_per_expert=[7921, 0, 0, 0, 0, 0, 0, 7386, 0, 0, 106, 0, 0, 0, 0, 24, 0, 0, 12, 0, 98, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 25, 0, 0, 0, 0, 1, 66, 217, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 5499, 7612, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1173, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 163, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1616, 4, 8100, 0, 0, 0, 813, 0, 0, 0, 8, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 287, 26, 0, 0, 1, 0, 6, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16384, 16364, 15224, 16382, 16200, 1642, 11622] dup_rows_per_dst=[14490, 272, 174, 0, 2676, 3240, 0, 18] combine_rows_per_src=[16384, 16384, 16364, 15224, 16382, 16200, 1642, 11622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=80194 payload_rows_remote=63810 dup_rows=50878 dup_rows_remote=40244 dedup_ratio=0.388168 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 1, 0, 387, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8161, 7958, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 45, 0, 0, 240, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7225, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 7960, 0, 0, 0, 8124, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 821, 0, 3, 0, 0] payload_rows_per_dst=[774, 14450, 0, 13840, 2382, 15980, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 6886, 564, 2544, 30248, 10634] combine_rows_per_src=[774, 14450, 0, 13840, 2382, 15980, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113176 payload_rows_remote=96792 dup_rows=17896 dup_rows_remote=3772 dedup_ratio=0.136536 router_tokens_per_expert=[8160, 0, 0, 0, 13, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 0, 6748, 0, 6, 0, 0, 329, 0, 2, 8163, 0, 0, 0, 38, 4197, 4430, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 1541, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8148, 0, 8, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 1, 6993, 0, 0, 0, 8151, 0, 0, 90, 0, 8081, 5, 0, 5, 0, 0, 0, 0, 0, 8, 0, 0, 0, 14, 0, 0, 0, 0, 7, 5, 3, 0, 0, 1] payload_rows_per_dst=[16348, 16356, 16384, 8954, 16224, 14064, 16384, 8462] dup_rows_per_dst=[2, 502, 14124, 8, 3116, 18, 118, 8] combine_rows_per_src=[16348, 16356, 16384, 8954, 16224, 14064, 16384, 8462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108306 payload_rows_remote=102840 dup_rows=22766 dup_rows_remote=20750 dedup_ratio=0.173691 router_tokens_per_expert=[8178, 0, 0, 0, 0, 0, 0, 3508, 22, 24, 4, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 5, 0, 0, 0, 0, 0, 6, 0, 579, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8176, 7678, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 83, 1142, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 1, 8053, 0, 0, 0, 2519, 15, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 0, 0, 0, 0, 0, 1, 0, 626, 2, 114, 0, 0] payload_rows_per_dst=[8256, 16384, 13928, 15366, 16376, 16146, 5466, 16384] dup_rows_per_dst=[74, 1180, 4384, 10762, 2686, 0, 2016, 1664] combine_rows_per_src=[8256, 16384, 13928, 15366, 16376, 16146, 5466, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127836 payload_rows_remote=113478 dup_rows=3236 dup_rows_remote=3236 dedup_ratio=0.024689 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 5, 0, 1128, 0, 0, 4, 3, 3, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8183, 0, 0, 0, 0, 8146, 7170, 2, 1, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 41, 107, 0, 0, 0, 0, 0, 218, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 8085, 4, 0, 0, 7793, 0, 0, 1, 0, 8149, 0, 0, 0, 0, 1, 0, 0, 61, 0, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16272, 16376, 14358, 16378, 16174, 15598, 16296] dup_rows_per_dst=[2438, 14, 2, 0, 652, 0, 0, 130] combine_rows_per_src=[16384, 16272, 16376, 14358, 16378, 16174, 15598, 16296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=768573440 dispatch_meta_bytes=448056 combine_payload_bytes=768573440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=769021496 ingress_bytes=844070912 link_bytes=844070912 token_bytes=1537594936 weight_movement_bytes=75497472 total_nvlink_bytes=1613092408 weight_to_token_ratio=0.049101 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.045956 gemm_active_groups=12 gemm_peak_group_rows=57856 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.070400728e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122472 payload_rows_remote=106088 dup_rows=8600 dup_rows_remote=2434 dedup_ratio=0.065613 router_tokens_per_expert=[8031, 0, 0, 148, 0, 0, 0, 0, 0, 3009, 0, 0, 5, 0, 82, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 5, 0, 2, 0, 0, 0, 44, 0, 41, 11, 0, 0, 8169, 0, 13, 2, 9, 8039, 8171, 0, 0, 0, 0, 0, 0, 54, 0, 20, 0, 0, 0, 0, 0, 0, 0, 534, 117, 0, 271, 0, 0, 0, 0, 1, 0, 0, 2, 7537, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7580, 0, 0, 2, 5299, 0, 0, 96, 0, 7917, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 1, 0, 204, 0] payload_rows_per_dst=[16384, 15778, 16384, 16384, 16376, 14038, 10812, 16316] dup_rows_per_dst=[6166, 606, 186, 128, 1308, 0, 4, 202] combine_rows_per_src=[16384, 15778, 16384, 16384, 16376, 14038, 10812, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=522731520 dispatch_meta_bytes=416216 combine_payload_bytes=522731520 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=523147736 ingress_bytes=673726464 link_bytes=673726464 token_bytes=1045879256 weight_movement_bytes=150994944 total_nvlink_bytes=1196874200 weight_to_token_ratio=0.144371 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=58496 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.543799905e-05 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=792920064 dispatch_meta_bytes=402256 combine_payload_bytes=792920064 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=793322320 ingress_bytes=943915008 link_bytes=943915008 token_bytes=1586242384 weight_movement_bytes=150994944 total_nvlink_bytes=1737237328 weight_to_token_ratio=0.095190 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=55808 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.197017096e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93618 payload_rows_remote=89358 dup_rows=37454 dup_rows_remote=37358 dedup_ratio=0.285751 router_tokens_per_expert=[0, 8181, 0, 0, 8117, 75, 0, 0, 2, 0, 0, 8076, 8151, 0, 0, 0, 2077, 2, 0, 3, 0, 0, 0, 0, 1, 2, 0, 0, 14, 39, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7846, 2, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 11, 0, 6726, 598, 0, 2, 4652, 1, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 7623, 0, 1, 0, 0, 0, 282, 0, 1, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 23, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 479, 233, 2131] payload_rows_per_dst=[16382, 4260, 15702, 14656, 9516, 11506, 16366, 5230] dup_rows_per_dst=[24770, 96, 11764, 40, 8, 182, 0, 594] combine_rows_per_src=[16382, 4260, 15702, 14656, 9516, 11506, 16366, 5230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=842465280 dispatch_meta_bytes=494360 combine_payload_bytes=842465280 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=842959640 ingress_bytes=993460224 link_bytes=993460224 token_bytes=1685424920 weight_movement_bytes=150994944 total_nvlink_bytes=1836419864 weight_to_token_ratio=0.089589 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=48896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.259847404e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=929611776 dispatch_meta_bytes=466856 combine_payload_bytes=929611776 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=930078632 ingress_bytes=1156104192 link_bytes=1156104192 token_bytes=1859690408 weight_movement_bytes=226492416 total_nvlink_bytes=2086182824 weight_to_token_ratio=0.121790 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=53120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.466102849e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102700 payload_rows_remote=86602 dup_rows=28372 dup_rows_remote=28372 dedup_ratio=0.216461 router_tokens_per_expert=[0, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8150, 8190, 0, 0, 0, 1597, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 2390, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 8192, 6, 0, 0, 5010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6756] payload_rows_per_dst=[0, 16384, 4782, 16384, 16300, 16098, 16384, 16368] dup_rows_per_dst=[0, 4780, 22, 38, 10020, 0, 0, 13512] combine_rows_per_src=[0, 16384, 4782, 16384, 16300, 16098, 16384, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=869072896 dispatch_meta_bytes=434088 combine_payload_bytes=869072896 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=869506984 ingress_bytes=1095565312 link_bytes=1095565312 token_bytes=1738579880 weight_movement_bytes=226492416 total_nvlink_bytes=1965072296 weight_to_token_ratio=0.130274 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.044118 gemm_active_groups=13 gemm_peak_group_rows=49280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.389331028e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=732020736 dispatch_meta_bytes=506864 combine_payload_bytes=732020736 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=732527600 ingress_bytes=807518208 link_bytes=807518208 token_bytes=1464548336 weight_movement_bytes=75497472 total_nvlink_bytes=1540045808 weight_to_token_ratio=0.051550 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.038603 gemm_active_groups=15 gemm_peak_group_rows=48896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.024046755e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=709443584 dispatch_meta_bytes=459896 combine_payload_bytes=709443584 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=709903480 ingress_bytes=860438528 link_bytes=860438528 token_bytes=1419347064 weight_movement_bytes=150994944 total_nvlink_bytes=1570342008 weight_to_token_ratio=0.106383 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=50944 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.091157169e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112406 payload_rows_remote=96336 dup_rows=18666 dup_rows_remote=3976 dedup_ratio=0.142410 router_tokens_per_expert=[0, 1, 0, 14, 0, 8, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 97, 0, 354, 0, 0, 0, 0, 0, 4, 198, 0, 0, 6964, 0, 0, 25, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 82, 0, 0, 9, 0, 0, 0, 0, 0, 3, 0, 6009, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1346, 0, 0, 0, 0, 8014, 0, 0, 2216, 0, 12, 0, 7309, 1, 0, 0, 0, 0, 213, 336, 0, 0, 0, 0, 0, 0, 8192, 0, 8156, 1, 0, 7982, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 5, 0, 108, 0, 0, 0, 7760, 0, 0] payload_rows_per_dst=[16326, 1154, 14000, 16384, 16070, 16322, 16384, 15766] dup_rows_per_dst=[42, 152, 154, 190, 14690, 3180, 258, 0] combine_rows_per_src=[16326, 1154, 14000, 16384, 16070, 16322, 16384, 15766] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=92986 payload_rows_remote=76610 dup_rows=38086 dup_rows_remote=38086 dedup_ratio=0.290573 router_tokens_per_expert=[0, 3025, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 3242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 1346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4554, 0, 21, 0, 0, 385, 2, 0, 0, 0, 0, 468, 0, 0, 2, 0, 310, 0, 89, 0, 0, 8192, 0, 0, 394, 0, 0, 95, 2675, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7948, 0, 8190, 0, 19, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0] payload_rows_per_dst=[9596, 6484, 16368, 9150, 2268, 16384, 16360, 16376] dup_rows_per_dst=[784, 0, 2698, 0, 244, 6332, 28028, 0] combine_rows_per_src=[9596, 6484, 16368, 9150, 2268, 16384, 16360, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98638 payload_rows_remote=88196 dup_rows=32434 dup_rows_remote=32272 dedup_ratio=0.247452 router_tokens_per_expert=[0, 21, 0, 0, 0, 67, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3936, 0, 0, 3, 0, 0, 0, 2, 11, 0, 4988, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 1, 0, 207, 0, 101, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 7, 0, 2, 0, 0, 0, 7498, 0, 0, 7, 1, 0, 0, 1, 0, 131, 8104, 13, 0, 0, 0, 0, 118, 7275, 0, 57, 0, 0, 6, 0, 2, 0, 0, 7, 0, 0, 0, 8036, 0, 8175, 1, 0, 16, 23, 0, 0, 18, 0, 0, 0, 0, 0, 10, 331, 0, 0, 0, 0, 0, 8121, 29, 0] payload_rows_per_dst=[16384, 7902, 10442, 16384, 16242, 14868, 96, 16320] dup_rows_per_dst=[160, 2, 162, 16080, 14998, 332, 2, 698] combine_rows_per_src=[16384, 7902, 10442, 16384, 16242, 14868, 96, 16320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108838 payload_rows_remote=97168 dup_rows=22234 dup_rows_remote=20718 dedup_ratio=0.169632 router_tokens_per_expert=[0, 0, 0, 0, 0, 1413, 0, 0, 0, 11, 0, 0, 0, 0, 3, 0, 18, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 6, 1152, 0, 139, 7645, 0, 0, 2, 0, 0, 0, 0, 0, 9, 30, 0, 0, 0, 0, 0, 0, 0, 72, 0, 14, 2, 0, 44, 0, 0, 1, 0, 0, 103, 0, 0, 284, 0, 653, 0, 0, 237, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8175, 3, 0, 5555, 0, 2, 4, 838, 0, 0, 0, 0, 0, 768, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 8185, 0, 0, 5835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 3, 0, 0, 0, 0, 8047, 11, 0] payload_rows_per_dst=[16378, 16314, 15370, 14860, 1760, 16368, 11670, 16118] dup_rows_per_dst=[2846, 2558, 2, 320, 588, 14330, 1516, 74] combine_rows_per_src=[16378, 16314, 15370, 14860, 1760, 16368, 11670, 16118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=107708 payload_rows_remote=91376 dup_rows=23364 dup_rows_remote=23330 dedup_ratio=0.178253 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 50, 0, 5, 0, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 7677, 2, 0, 2, 0, 0, 0, 3894, 0, 0, 6577, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 17, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 13, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 8185, 14, 0, 5589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1] payload_rows_per_dst=[16384, 16250, 13870, 16332, 16384, 918, 11206, 16364] dup_rows_per_dst=[106, 6900, 14, 34, 16310, 0, 0, 0] combine_rows_per_src=[16384, 16250, 13870, 16332, 16384, 918, 11206, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108356 payload_rows_remote=92120 dup_rows=22716 dup_rows_remote=22714 dedup_ratio=0.173309 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 1283, 0, 0, 182, 0, 0, 0, 6479, 2, 0, 7493, 0, 0, 0, 50, 2, 0, 0, 0, 0, 0, 0, 9, 4161, 0, 0, 0, 0, 0, 0, 16, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 456, 0, 0, 21, 0, 8000, 0, 0, 0, 0, 0, 4899, 10, 0, 0, 17, 31, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8054, 0, 8103, 0, 0, 7686, 0, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0] payload_rows_per_dst=[16236, 16364, 16384, 16140, 3464, 10508, 12878, 16382] dup_rows_per_dst=[2, 12662, 8964, 152, 478, 388, 6, 64] combine_rows_per_src=[16236, 16364, 16384, 16140, 3464, 10508, 12878, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=789184512 dispatch_meta_bytes=401248 combine_payload_bytes=789184512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=789585760 ingress_bytes=864681984 link_bytes=864681984 token_bytes=1578770272 weight_movement_bytes=75497472 total_nvlink_bytes=1654267744 weight_to_token_ratio=0.047820 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.040441 gemm_active_groups=16 gemm_peak_group_rows=34816 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.096538469e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627589120 dispatch_meta_bytes=458784 combine_payload_bytes=627589120 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=628047904 ingress_bytes=703086592 link_bytes=703086592 token_bytes=1255637024 weight_movement_bytes=75497472 total_nvlink_bytes=1331134496 weight_to_token_ratio=0.060127 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916127656e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130436 payload_rows_remote=114082 dup_rows=636 dup_rows_remote=500 dedup_ratio=0.004852 router_tokens_per_expert=[8171, 0, 0, 0, 1, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 4, 0, 0, 5, 0, 0, 0, 0, 0, 65, 0, 2, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8133, 0, 0, 0, 43, 8033, 8175, 1, 5, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 27, 20, 0, 13, 0, 0, 0, 0, 0, 0, 59, 8153, 0, 21, 0, 0, 1, 1, 0, 0, 0, 3, 0, 0, 87, 0, 8107, 0, 15, 0, 8007, 0, 0, 0, 0, 8086, 0, 0, 0, 10, 0, 0, 0, 0, 135, 0, 0, 0, 35, 0, 2, 0, 0, 0, 0, 17, 0, 0, 0] payload_rows_per_dst=[16344, 16354, 16270, 16384, 16330, 16336, 16062, 16356] dup_rows_per_dst=[78, 136, 6, 132, 90, 104, 2, 88] combine_rows_per_src=[16344, 16354, 16270, 16384, 16330, 16336, 16062, 16356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=722501632 dispatch_meta_bytes=481872 combine_payload_bytes=722501632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=722983504 ingress_bytes=797999104 link_bytes=797999104 token_bytes=1445485136 weight_movement_bytes=75497472 total_nvlink_bytes=1520982608 weight_to_token_ratio=0.052230 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.042279 gemm_active_groups=14 gemm_peak_group_rows=57856 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.011975191e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=796000256 dispatch_meta_bytes=471544 combine_payload_bytes=796000256 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=796471800 ingress_bytes=1173487616 link_bytes=1173487616 token_bytes=1592472056 weight_movement_bytes=377487360 total_nvlink_bytes=1969959416 weight_to_token_ratio=0.237045 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=50176 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.488147478e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126934 payload_rows_remote=110706 dup_rows=4138 dup_rows_remote=4138 dedup_ratio=0.031570 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8131, 6323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1763, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 8161, 0, 0, 0, 0, 8000, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 12646, 16276, 16228, 16324, 16310] dup_rows_per_dst=[564, 2, 106, 0, 3256, 0, 12, 198] combine_rows_per_src=[16384, 16382, 16384, 12646, 16276, 16228, 16324, 16310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=748552192 dispatch_meta_bytes=458824 combine_payload_bytes=748552192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=749011016 ingress_bytes=824049664 link_bytes=824049664 token_bytes=1497563208 weight_movement_bytes=75497472 total_nvlink_bytes=1573060680 weight_to_token_ratio=0.050414 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.044118 gemm_active_groups=13 gemm_peak_group_rows=51584 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.045010968e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=754647040 dispatch_meta_bytes=459336 combine_payload_bytes=754647040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=755106376 ingress_bytes=830144512 link_bytes=830144512 token_bytes=1509753416 weight_movement_bytes=75497472 total_nvlink_bytes=1585250888 weight_to_token_ratio=0.050006 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.042279 gemm_active_groups=13 gemm_peak_group_rows=59264 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.052740093e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=934559744 dispatch_meta_bytes=458328 combine_payload_bytes=934559744 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=935018072 ingress_bytes=1085554688 link_bytes=1085554688 token_bytes=1869577816 weight_movement_bytes=150994944 total_nvlink_bytes=2020572760 weight_to_token_ratio=0.080764 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.034926 gemm_active_groups=17 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.376636148e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=906903552 dispatch_meta_bytes=459376 combine_payload_bytes=906903552 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=907362928 ingress_bytes=1133395968 link_bytes=1133395968 token_bytes=1814266480 weight_movement_bytes=226492416 total_nvlink_bytes=2040758896 weight_to_token_ratio=0.124840 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=62464 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.437305625e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127318 payload_rows_remote=110938 dup_rows=3754 dup_rows_remote=3446 dedup_ratio=0.028641 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 7784, 0, 0, 0, 0, 0, 7778, 0, 0, 0, 0, 1721, 157, 0, 8, 0, 0, 0, 96, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 107, 4, 0, 0, 0, 0, 287, 0, 1, 0, 0, 0, 0, 0, 14, 0, 0, 17, 8025, 0, 392, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7720, 0, 0, 0, 0, 0, 0, 10, 2, 1, 0, 9, 8192, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6850, 0, 0, 0, 0, 0] payload_rows_per_dst=[16302, 16384, 16370, 16384, 16380, 16314, 15478, 13706] dup_rows_per_dst=[0, 70, 2958, 366, 308, 46, 6, 0] combine_rows_per_src=[16302, 16384, 16370, 16384, 16380, 16314, 15478, 13706] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108830 payload_rows_remote=92446 dup_rows=22242 dup_rows_remote=15976 dedup_ratio=0.169693 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7607, 0, 0, 0, 0, 0, 7116, 0, 0, 0, 0, 0, 0, 799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8158, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4834, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 165, 0, 0, 0, 0, 0, 0, 0, 292, 3752, 0, 0, 0, 0] payload_rows_per_dst=[16372, 15692, 1598, 16380, 16382, 16344, 9678, 16384] dup_rows_per_dst=[2, 4, 0, 0, 0, 15284, 686, 6266] combine_rows_per_src=[16372, 15692, 1598, 16380, 16382, 16344, 9678, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120650 payload_rows_remote=104350 dup_rows=10422 dup_rows_remote=5718 dedup_ratio=0.079514 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 11, 0, 0, 0, 0, 7714, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 15, 0, 0, 0, 59, 0, 0, 0, 0, 8040, 350, 0, 0, 1, 0, 7389, 0, 0, 102, 0, 0, 0, 0, 5, 0, 0, 0, 2, 152, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 578, 28, 0, 0, 0, 0, 282, 0, 0, 0, 7678, 0, 0, 4, 0, 2934, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 6744, 0, 0, 0, 17, 0, 0, 88, 0, 1, 0, 0, 8143, 0, 0, 0, 4, 1451, 0, 0, 0, 0, 73, 0, 0, 28, 5363, 0, 0, 0, 128] payload_rows_per_dst=[15450, 16382, 16300, 16180, 16378, 13692, 13690, 12578] dup_rows_per_dst=[4, 114, 4704, 304, 754, 3016, 10, 1516] combine_rows_per_src=[15450, 16382, 16300, 16180, 16378, 13692, 13690, 12578] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112704 payload_rows_remote=96322 dup_rows=18368 dup_rows_remote=10680 dedup_ratio=0.140137 router_tokens_per_expert=[1, 0, 0, 17, 0, 0, 0, 1, 0, 0, 0, 0, 8144, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 43, 0, 0, 32, 6744, 0, 2, 0, 0, 0, 5729, 0, 0, 0, 0, 396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 1023, 0, 0, 0, 0, 0, 0, 0, 42, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 91, 0, 0, 278, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 13, 0, 6, 14, 0, 0, 0, 8175, 6, 18, 15, 0, 25, 0, 0, 0, 0, 0, 0, 0, 3478, 6760, 0, 0, 0, 13] payload_rows_per_dst=[16322, 16384, 796, 16362, 16344, 14212, 16382, 15902] dup_rows_per_dst=[4, 160, 0, 1904, 76, 22, 7688, 8514] combine_rows_per_src=[16322, 16384, 796, 16362, 16344, 14212, 16382, 15902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123518 payload_rows_remote=107136 dup_rows=7554 dup_rows_remote=7480 dedup_ratio=0.057632 router_tokens_per_expert=[0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 11, 0, 8166, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 3, 0, 5402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 7, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 1379, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 8192, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 136, 0, 0, 0, 0, 0, 0, 0, 6298, 3207, 0, 0, 0, 1] payload_rows_per_dst=[16354, 16384, 10806, 16382, 16382, 16236, 16384, 14590] dup_rows_per_dst=[4, 6, 4, 74, 2, 2720, 46, 4698] combine_rows_per_src=[16354, 16384, 10806, 16382, 16382, 16236, 16384, 14590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=908804096 dispatch_meta_bytes=457536 combine_payload_bytes=908804096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=909261632 ingress_bytes=984301568 link_bytes=984301568 token_bytes=1818065728 weight_movement_bytes=75497472 total_nvlink_bytes=1893563200 weight_to_token_ratio=0.041526 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.248232939e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=107838 payload_rows_remote=91462 dup_rows=23234 dup_rows_remote=22102 dedup_ratio=0.177261 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 121, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 85, 8046, 123, 1, 0, 40, 0, 6049, 0, 2369, 29, 1, 1, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 69, 0, 0, 0, 2, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5877, 0, 0, 4, 3, 99, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 11, 8, 0, 0, 0, 4, 0, 0, 8164, 0, 2, 13, 1, 237, 0, 0, 0, 18, 0, 0, 0, 8015, 1549, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16342, 16384, 14314, 11874, 230, 16176, 16142] dup_rows_per_dst=[1132, 162, 18392, 0, 2, 14, 8, 3524] combine_rows_per_src=[16376, 16342, 16384, 14314, 11874, 230, 16176, 16142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120266 payload_rows_remote=104074 dup_rows=10806 dup_rows_remote=10642 dedup_ratio=0.082443 router_tokens_per_expert=[0, 522, 0, 0, 0, 0, 167, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 25, 0, 0, 8072, 0, 16, 0, 0, 0, 0, 62, 0, 0, 7881, 0, 0, 13, 0, 0, 0, 1, 31, 0, 0, 14, 0, 315, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 363, 0, 2, 0, 0, 0, 1, 0, 4257, 0, 0, 0, 0, 0, 2, 1, 0, 124, 0, 0, 0, 8, 7, 60, 0, 0, 8057, 0, 0, 88, 0, 0, 34, 7467, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3648, 49, 8058, 0, 0, 8104, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8057, 0, 0] payload_rows_per_dst=[16384, 16192, 15904, 8022, 16380, 14960, 16306, 16118] dup_rows_per_dst=[1118, 164, 606, 36, 8652, 218, 12, 0] combine_rows_per_src=[16384, 16192, 15904, 8022, 16380, 14960, 16306, 16118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=757317632 dispatch_meta_bytes=433688 combine_payload_bytes=757317632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=757751320 ingress_bytes=832815104 link_bytes=832815104 token_bytes=1515068952 weight_movement_bytes=75497472 total_nvlink_bytes=1590566424 weight_to_token_ratio=0.049831 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=29184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.056126779e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=854835200 dispatch_meta_bytes=440272 combine_payload_bytes=854835200 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=855275472 ingress_bytes=1232322560 link_bytes=1232322560 token_bytes=1710110672 weight_movement_bytes=377487360 total_nvlink_bytes=2087598032 weight_to_token_ratio=0.220739 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.042279 gemm_active_groups=15 gemm_peak_group_rows=38528 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.562758469e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=789069824 dispatch_meta_bytes=428008 combine_payload_bytes=789069824 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=789497832 ingress_bytes=940064768 link_bytes=940064768 token_bytes=1578567656 weight_movement_bytes=150994944 total_nvlink_bytes=1729562600 weight_to_token_ratio=0.095653 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.192134450e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114286 payload_rows_remote=99570 dup_rows=16786 dup_rows_remote=15538 dedup_ratio=0.128067 router_tokens_per_expert=[0, 1, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 6, 0, 7880, 0, 7, 0, 0, 0, 0, 6094, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 375, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 301, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 179, 412, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 8100, 0, 8192, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 204, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 15798, 16134, 16352, 2168, 14716, 16350, 16384] dup_rows_per_dst=[1942, 12176, 0, 636, 376, 1248, 0, 408] combine_rows_per_src=[16384, 15798, 16134, 16352, 2168, 14716, 16350, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=877658112 dispatch_meta_bytes=458464 combine_payload_bytes=877658112 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=878116576 ingress_bytes=1028653056 link_bytes=1028653056 token_bytes=1755774688 weight_movement_bytes=150994944 total_nvlink_bytes=1906769632 weight_to_token_ratio=0.085999 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.049632 gemm_active_groups=10 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.304476869e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=852574208 dispatch_meta_bytes=458864 combine_payload_bytes=852574208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=853033072 ingress_bytes=928071680 link_bytes=928071680 token_bytes=1705607280 weight_movement_bytes=75497472 total_nvlink_bytes=1781104752 weight_to_token_ratio=0.044264 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.042279 gemm_active_groups=15 gemm_peak_group_rows=40576 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176925526e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=749256704 dispatch_meta_bytes=454256 combine_payload_bytes=749256704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=749710960 ingress_bytes=824754176 link_bytes=824754176 token_bytes=1498967664 weight_movement_bytes=75497472 total_nvlink_bytes=1574465136 weight_to_token_ratio=0.050366 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.045904388e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=815677440 dispatch_meta_bytes=460432 combine_payload_bytes=815677440 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=816137872 ingress_bytes=1042169856 link_bytes=1042169856 token_bytes=1631815312 weight_movement_bytes=226492416 total_nvlink_bytes=1858307728 weight_to_token_ratio=0.138798 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.040441 gemm_active_groups=16 gemm_peak_group_rows=26112 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.321618074e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=96856 payload_rows_remote=89104 dup_rows=34216 dup_rows_remote=34188 dedup_ratio=0.261047 router_tokens_per_expert=[0, 0, 5, 0, 102, 0, 713, 0, 0, 0, 0, 859, 0, 0, 301, 0, 1, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 7830, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 7519, 0, 0, 0, 0, 120, 7991, 8005, 0, 0, 3624, 40, 0, 0, 0, 55, 0, 0, 0, 15, 156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 3990, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8012, 0, 0, 0, 0, 9, 0, 0, 28, 0, 0, 8160, 0, 0, 22, 0] payload_rows_per_dst=[16384, 30, 16000, 16334, 7752, 7988, 15990, 16378] dup_rows_per_dst=[3600, 2, 15364, 15162, 28, 0, 0, 60] combine_rows_per_src=[16384, 30, 16000, 16334, 7752, 7988, 15990, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104240 payload_rows_remote=87894 dup_rows=26832 dup_rows_remote=13952 dedup_ratio=0.204712 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1048, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 7215, 0, 3, 0, 2533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 92, 0, 0, 0, 0, 1, 0, 0, 645, 8089, 22, 0, 7, 113, 35, 0, 0, 0, 105, 0, 0, 0, 0, 7238, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 678, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 4849, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 4, 0] payload_rows_per_dst=[6104, 16368, 6182, 14274, 12304, 16382, 16280, 16346] dup_rows_per_dst=[2030, 16, 340, 226, 100, 1548, 9692, 12880] combine_rows_per_src=[6104, 16368, 6182, 14274, 12304, 16382, 16280, 16346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111598 payload_rows_remote=95222 dup_rows=19474 dup_rows_remote=19362 dedup_ratio=0.148575 router_tokens_per_expert=[0, 1416, 0, 1, 0, 0, 0, 39, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 9, 0, 0, 8128, 2, 0, 1, 0, 5, 0, 0, 0, 8177, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 4, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 8001, 5672, 0, 0, 660, 49, 0, 0, 0, 623, 0, 0, 0, 0, 381, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 140, 0, 137, 0, 7858, 0, 113, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 0, 10, 8153, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7620, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16354, 16376, 11454, 3322, 16224, 16228, 15260] dup_rows_per_dst=[2878, 16290, 112, 2, 104, 54, 34, 0] combine_rows_per_src=[16380, 16354, 16376, 11454, 3322, 16224, 16228, 15260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109470 payload_rows_remote=93242 dup_rows=21602 dup_rows_remote=21602 dedup_ratio=0.164810 router_tokens_per_expert=[1, 0, 0, 0, 0, 10, 0, 5, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 0, 68, 0, 0, 7378, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 3, 5, 1, 0, 127, 8, 0, 0, 0, 0, 0, 0, 0, 7638, 8113, 2157, 0, 0, 0, 0, 0, 38, 0, 3074, 0, 296, 0, 0, 3416, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 26, 0, 1, 0, 8, 0, 5, 0, 4, 7652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8166, 941, 0, 0, 0] payload_rows_per_dst=[16382, 150, 16370, 15986, 12636, 15362, 16228, 16356] dup_rows_per_dst=[24, 8, 14758, 3886, 1012, 32, 0, 1882] combine_rows_per_src=[16382, 150, 16370, 15986, 12636, 15362, 16228, 16356] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100936 payload_rows_remote=100738 dup_rows=30136 dup_rows_remote=30108 dedup_ratio=0.229919 router_tokens_per_expert=[0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 1, 0, 0, 0, 8127, 0, 4, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 15, 0, 0, 93, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 0, 1, 8179, 0, 22, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 14, 0, 8078, 17, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7670, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 529, 0, 0, 0] payload_rows_per_dst=[15348, 16324, 3746, 198, 16376, 16200, 16380, 16364] dup_rows_per_dst=[4, 12600, 8, 28, 16316, 32, 44, 1104] combine_rows_per_src=[15348, 16324, 3746, 198, 16376, 16200, 16380, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=729939968 dispatch_meta_bytes=493168 combine_payload_bytes=729939968 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=730433136 ingress_bytes=880934912 link_bytes=880934912 token_bytes=1460373104 weight_movement_bytes=150994944 total_nvlink_bytes=1611368048 weight_to_token_ratio=0.103395 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.044118 gemm_active_groups=13 gemm_peak_group_rows=36352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.117149469e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113626 payload_rows_remote=97248 dup_rows=17446 dup_rows_remote=17358 dedup_ratio=0.133102 router_tokens_per_expert=[0, 17, 0, 0, 0, 7, 4, 0, 0, 0, 26, 0, 8185, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 1, 1, 0, 0, 0, 7590, 0, 0, 37, 0, 0, 6693, 0, 5, 0, 0, 490, 0, 1446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8017, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 7436, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8175, 0, 0, 3, 0, 539, 7, 0, 0, 14, 0, 0, 8169, 0, 0, 0, 7, 95, 0, 0, 0, 0, 0, 0, 0, 0, 331, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16374, 16034, 14884, 16342, 16378, 852] dup_rows_per_dst=[94, 88, 16148, 0, 2, 0, 1098, 16] combine_rows_per_src=[16384, 16378, 16374, 16034, 14884, 16342, 16378, 852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=720027648 dispatch_meta_bytes=407384 combine_payload_bytes=720027648 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=720435032 ingress_bytes=946520064 link_bytes=946520064 token_bytes=1440462680 weight_movement_bytes=226492416 total_nvlink_bytes=1666955096 weight_to_token_ratio=0.157236 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=52480 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.200320674e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102084 payload_rows_remote=85756 dup_rows=28988 dup_rows_remote=27324 dedup_ratio=0.221161 router_tokens_per_expert=[0, 833, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1375, 0, 0, 0, 0, 0, 0, 3, 0, 6935, 0, 0, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 227, 8175, 1440, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 233, 0, 0, 8133, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 6721, 79, 0, 0, 0, 0, 4, 0, 1, 0, 0, 64, 0, 6370, 0, 0, 4, 8156, 0, 0, 0, 39, 0, 67, 0, 0, 1, 1, 8016, 4844, 0, 19, 0] payload_rows_per_dst=[16328, 13876, 374, 16360, 16382, 16036, 13010, 9718] dup_rows_per_dst=[1664, 2750, 2, 3328, 7544, 13442, 34, 224] combine_rows_per_src=[16328, 13876, 374, 16360, 16382, 16036, 13010, 9718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=780058624 dispatch_meta_bytes=458336 combine_payload_bytes=780058624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=780516960 ingress_bytes=855556096 link_bytes=855556096 token_bytes=1560575584 weight_movement_bytes=75497472 total_nvlink_bytes=1636073056 weight_to_token_ratio=0.048378 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=39424 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084965558e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=825245696 dispatch_meta_bytes=523384 combine_payload_bytes=825245696 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=825769080 ingress_bytes=976240640 link_bytes=976240640 token_bytes=1651014776 weight_movement_bytes=150994944 total_nvlink_bytes=1802009720 weight_to_token_ratio=0.091456 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.045956 gemm_active_groups=14 gemm_peak_group_rows=22144 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.238010548e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124460 payload_rows_remote=108076 dup_rows=6612 dup_rows_remote=6396 dedup_ratio=0.050446 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 7803, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 467, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8060, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 15, 1, 0, 0, 0, 0, 0, 0, 28, 8148, 0, 0, 0, 0] payload_rows_per_dst=[16272, 16384, 10212, 16384, 16382, 16384, 16122, 16320] dup_rows_per_dst=[0, 138, 26, 4, 4922, 216, 1242, 64] combine_rows_per_src=[16272, 16384, 10212, 16384, 16382, 16384, 16122, 16320] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=763838464 dispatch_meta_bytes=459376 combine_payload_bytes=763838464 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=764297840 ingress_bytes=839335936 link_bytes=839335936 token_bytes=1528136304 weight_movement_bytes=75497472 total_nvlink_bytes=1603633776 weight_to_token_ratio=0.049405 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.045956 gemm_active_groups=13 gemm_peak_group_rows=51456 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.064396112e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19224358760 prefetch_bytes=981467136 reduce_bytes=452984832 weight_movement_bytes=1434451968 egress_bytes=9614946152 ingress_bytes=11043864576 link_bytes=11043864576 total_nvlink_bytes=20658810728 weight_to_token_ratio=0.074616 prefetch_share_of_ingress=0.088870 dedup_ratio=0.152167 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=6 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.040288 gemm_inner_pad_upper_bound=22479 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.167097902e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=796655616 dispatch_meta_bytes=458424 combine_payload_bytes=796655616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=797114040 ingress_bytes=872153088 link_bytes=872153088 token_bytes=1593769656 weight_movement_bytes=75497472 total_nvlink_bytes=1669267128 weight_to_token_ratio=0.047370 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.045956 gemm_active_groups=12 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.106012880e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15784386336 prefetch_bytes=754974720 reduce_bytes=905969664 weight_movement_bytes=1660944384 egress_bytes=7894720288 ingress_bytes=9550610432 link_bytes=9550610432 total_nvlink_bytes=17445330720 weight_to_token_ratio=0.105227 prefetch_share_of_ingress=0.079050 dedup_ratio=0.266066 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=12 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041054 gemm_inner_pad_upper_bound=22352 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.009293198e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=702513152 dispatch_meta_bytes=452320 combine_payload_bytes=702513152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=702965472 ingress_bytes=778010624 link_bytes=778010624 token_bytes=1405478624 weight_movement_bytes=75497472 total_nvlink_bytes=1480976096 weight_to_token_ratio=0.053717 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.042279 gemm_active_groups=15 gemm_peak_group_rows=59520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866269845e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19153146536 prefetch_bytes=830472192 reduce_bytes=1132462080 weight_movement_bytes=1962934272 egress_bytes=9579336360 ingress_bytes=11536744448 link_bytes=11536744448 total_nvlink_bytes=21116080808 weight_to_token_ratio=0.102486 prefetch_share_of_ingress=0.071985 dedup_ratio=0.161641 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=15 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.040594 gemm_inner_pad_upper_bound=22987 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.219184657e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=885358592 dispatch_meta_bytes=457888 combine_payload_bytes=885358592 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=885816480 ingress_bytes=960856064 link_bytes=960856064 token_bytes=1771175072 weight_movement_bytes=75497472 total_nvlink_bytes=1846672544 weight_to_token_ratio=0.042626 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.218500740e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19226987136 prefetch_bytes=754974720 reduce_bytes=830472192 weight_movement_bytes=1585446912 egress_bytes=9616231040 ingress_bytes=11196203008 link_bytes=11196203008 total_nvlink_bytes=20812434048 weight_to_token_ratio=0.082459 prefetch_share_of_ingress=0.067431 dedup_ratio=0.148125 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=11 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=21336 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.183196783e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=17569876080 prefetch_bytes=830472192 reduce_bytes=754974720 weight_movement_bytes=1585446912 egress_bytes=8787626096 ingress_bytes=10367696896 link_bytes=10367696896 total_nvlink_bytes=19155322992 weight_to_token_ratio=0.090237 prefetch_share_of_ingress=0.080102 dedup_ratio=0.206838 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=10 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=22606 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.095641586e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=16995721088 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=8500420480 ingress_bytes=10156244992 link_bytes=10156244992 total_nvlink_bytes=18656665472 weight_to_token_ratio=0.097727 prefetch_share_of_ingress=0.081770 dedup_ratio=0.218211 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041667 gemm_inner_pad_upper_bound=22225 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.073295687e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108156 payload_rows_remote=91798 dup_rows=22916 dup_rows_remote=10812 dedup_ratio=0.174835 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 542, 0, 0, 19, 0, 0, 0, 0, 0, 0, 5379, 0, 838, 0, 0, 0, 0, 0, 23, 7991, 0, 0, 19, 7635, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 74, 0, 0, 63, 0, 0, 0, 6305, 8171, 15, 0, 0, 0, 1, 0, 0, 0, 0, 7, 272, 0, 0, 8187, 0, 0, 5, 0, 0, 42, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 1926, 1663, 0, 0, 0, 0, 0, 0, 1, 98, 0, 1, 5, 0, 1, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7962, 66, 10, 0, 74] payload_rows_per_dst=[16290, 16358, 15306, 14804, 16384, 11824, 8996, 8194] dup_rows_per_dst=[1082, 12104, 4, 8790, 560, 170, 206, 0] combine_rows_per_src=[16290, 16358, 15306, 14804, 16384, 11824, 8996, 8194] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115522 payload_rows_remote=99744 dup_rows=15550 dup_rows_remote=15550 dedup_ratio=0.118637 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3921, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 4938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 321, 8192, 4799, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 3, 0, 0, 1934, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7886, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 783, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 2, 0, 0, 0] payload_rows_per_dst=[16384, 7842, 16306, 10138, 16384, 15778, 16384, 16306] dup_rows_per_dst=[190, 0, 9824, 102, 3868, 0, 1566, 0] combine_rows_per_src=[16384, 7842, 16306, 10138, 16384, 15778, 16384, 16306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=752009216 dispatch_meta_bytes=410440 combine_payload_bytes=752009216 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=752419656 ingress_bytes=978501632 link_bytes=978501632 token_bytes=1504428872 weight_movement_bytes=226492416 total_nvlink_bytes=1730921288 weight_to_token_ratio=0.150550 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=35200 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.240877804e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=817102848 dispatch_meta_bytes=461176 combine_payload_bytes=817102848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=817564024 ingress_bytes=892600320 link_bytes=892600320 token_bytes=1634666872 weight_movement_bytes=75497472 total_nvlink_bytes=1710164344 weight_to_token_ratio=0.046185 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=52352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.131942849e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=18318802992 prefetch_bytes=981467136 reduce_bytes=981467136 weight_movement_bytes=1962934272 egress_bytes=9162047536 ingress_bytes=11119689728 link_bytes=11119689728 total_nvlink_bytes=20281737264 weight_to_token_ratio=0.107154 prefetch_share_of_ingress=0.088264 dedup_ratio=0.174600 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=13 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.040441 gemm_inner_pad_upper_bound=23241 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.175110982e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19916269376 prefetch_bytes=905969664 reduce_bytes=981467136 weight_movement_bytes=1887436800 egress_bytes=9960908608 ingress_bytes=11842797568 link_bytes=11842797568 total_nvlink_bytes=21803706176 weight_to_token_ratio=0.094769 prefetch_share_of_ingress=0.076500 dedup_ratio=0.116350 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=13 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041513 gemm_inner_pad_upper_bound=22352 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.251527860e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none + [2026-08-29 23:35:10] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 906.0 | throughput per GPU (TFLOP/s/GPU): 199.6 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.685875E+00 | loss scale: 1.0 | grad norm: 62.849 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100848 payload_rows_remote=86988 dup_rows=30224 dup_rows_remote=21786 dedup_ratio=0.230591 router_tokens_per_expert=[311, 1, 0, 87, 105, 327, 544, 0, 1, 1001, 480, 47, 191, 50, 73, 158, 1, 1092, 0, 16, 1036, 0, 1, 1045, 2133, 6, 0, 7783, 4944, 6158, 12, 863, 108, 303, 179, 1, 21, 1026, 0, 6, 46, 0, 363, 1651, 0, 230, 4, 7928, 1, 14, 7, 2, 0, 73, 0, 5, 173, 0, 20, 351, 0, 0, 9, 36, 2174, 1, 0, 0, 778, 0, 13, 96, 452, 1, 1116, 889, 52, 2, 160, 7, 0, 2, 0, 0, 28, 0, 0, 523, 1, 124, 7, 33, 10, 15, 27, 6, 33, 143, 2, 10, 10, 4, 24, 6, 270, 402, 196, 1251, 0, 51, 1950, 11, 1, 3936, 0, 6, 0, 42, 1009, 7, 0, 26, 8151, 331, 2, 13, 2, 107] payload_rows_per_dst=[16336, 13860, 16014, 15642, 9176, 12710, 7596, 9514] dup_rows_per_dst=[6718, 8438, 7718, 1306, 2306, 1158, 1130, 1450] combine_rows_per_src=[16336, 13860, 16014, 15642, 9176, 12710, 7596, 9514] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102496 payload_rows_remote=86690 dup_rows=28576 dup_rows_remote=21264 dedup_ratio=0.218018 router_tokens_per_expert=[615, 1, 0, 171, 58, 161, 469, 0, 0, 826, 229, 19, 113, 43, 71, 63, 6, 875, 1, 24, 776, 3, 1, 889, 1694, 12, 0, 7435, 4915, 6605, 8, 971, 127, 225, 328, 0, 16, 570, 0, 51, 147, 0, 558, 1743, 0, 76, 7, 7711, 1, 53, 11, 1, 0, 28, 2, 3, 133, 3, 24, 218, 1, 4, 38, 222, 2670, 1, 6, 0, 486, 0, 44, 14, 585, 0, 1163, 1047, 52, 8, 82, 4, 2, 7, 1, 3, 30, 0, 3, 571, 1, 79, 24, 33, 15, 11, 15, 0, 0, 107, 0, 6, 31, 11, 73, 13, 142, 1036, 204, 1384, 0, 45, 2838, 6, 13, 3648, 0, 5, 0, 69, 1000, 2, 3, 38, 7959, 427, 9, 5, 6, 144] payload_rows_per_dst=[16116, 13302, 15806, 15156, 9676, 13570, 9744, 9126] dup_rows_per_dst=[5480, 7048, 7312, 1198, 2648, 1230, 2048, 1612] combine_rows_per_src=[16116, 13302, 15806, 15156, 9676, 13570, 9744, 9126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93208 payload_rows_remote=76882 dup_rows=37864 dup_rows_remote=26468 dedup_ratio=0.288879 router_tokens_per_expert=[412, 2, 0, 65, 92, 300, 431, 0, 5, 534, 225, 10, 129, 90, 77, 155, 6, 621, 0, 34, 765, 0, 14, 1199, 3198, 4, 0, 7379, 4198, 6096, 29, 488, 65, 323, 171, 0, 5, 804, 0, 46, 101, 1, 445, 1333, 0, 143, 14, 7774, 0, 10, 6, 0, 0, 25, 1, 10, 95, 0, 37, 297, 1, 2, 4, 44, 2406, 0, 0, 0, 587, 0, 11, 108, 814, 0, 1662, 1730, 53, 21, 233, 1, 1, 1, 1, 3, 57, 0, 0, 612, 0, 102, 10, 5, 39, 8, 62, 0, 18, 131, 0, 3, 18, 3, 13, 6, 103, 671, 390, 1480, 0, 140, 2129, 10, 2, 3097, 0, 4, 1, 93, 1590, 2, 1, 65, 8060, 509, 6, 11, 6, 132] payload_rows_per_dst=[4638, 10056, 6036, 14868, 10996, 14684, 15604, 16326] dup_rows_per_dst=[416, 2660, 866, 954, 4256, 7706, 9610, 11396] combine_rows_per_src=[4638, 10056, 6036, 14868, 10996, 14684, 15604, 16326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99082 payload_rows_remote=83994 dup_rows=31990 dup_rows_remote=29812 dedup_ratio=0.244064 router_tokens_per_expert=[587, 0, 0, 129, 105, 164, 578, 0, 0, 571, 241, 30, 145, 68, 266, 132, 14, 1033, 0, 14, 961, 0, 0, 863, 1334, 35, 0, 7398, 4446, 5910, 6, 1161, 277, 335, 310, 2, 48, 1368, 0, 1, 76, 0, 433, 1595, 0, 235, 30, 7722, 0, 47, 3, 0, 1, 52, 1, 7, 195, 1, 13, 136, 2, 1, 4, 24, 2533, 1, 1, 0, 550, 3, 16, 65, 639, 0, 1870, 1198, 89, 2, 174, 8, 0, 1, 0, 9, 34, 0, 1, 617, 3, 131, 17, 4, 17, 4, 52, 2, 4, 296, 0, 12, 22, 12, 64, 16, 469, 255, 286, 1564, 0, 143, 2440, 18, 0, 2576, 0, 8, 0, 139, 1477, 6, 1, 2, 8084, 387, 9, 10, 1, 84] payload_rows_per_dst=[16272, 10242, 15836, 15088, 12152, 12330, 9308, 7854] dup_rows_per_dst=[5928, 4334, 9028, 2178, 5808, 1274, 1894, 1546] combine_rows_per_src=[16272, 10242, 15836, 15088, 12152, 12330, 9308, 7854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=94450 payload_rows_remote=78822 dup_rows=36622 dup_rows_remote=26956 dedup_ratio=0.279404 router_tokens_per_expert=[804, 5, 0, 58, 32, 222, 633, 0, 1, 562, 591, 24, 394, 51, 158, 207, 30, 1090, 0, 12, 860, 8, 87, 983, 1505, 11, 1, 7632, 4507, 5646, 8, 673, 81, 434, 190, 3, 7, 1382, 1, 109, 142, 4, 497, 1664, 0, 121, 21, 7473, 0, 17, 4, 7, 1, 74, 0, 0, 88, 4, 15, 174, 2, 5, 15, 38, 2640, 1, 16, 0, 556, 1, 7, 139, 551, 0, 1303, 1171, 56, 11, 147, 12, 0, 21, 2, 23, 126, 0, 1, 879, 0, 190, 49, 113, 36, 8, 59, 9, 11, 115, 3, 9, 99, 33, 42, 14, 172, 793, 271, 1766, 3, 71, 1746, 26, 2, 3472, 0, 17, 1, 86, 730, 20, 7, 30, 7972, 389, 32, 11, 12, 86] payload_rows_per_dst=[6284, 8526, 7434, 15396, 10672, 14354, 15628, 16156] dup_rows_per_dst=[1200, 2010, 1878, 756, 3656, 7878, 9666, 9578] combine_rows_per_src=[6284, 8526, 7434, 15396, 10672, 14354, 15628, 16156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99880 payload_rows_remote=85140 dup_rows=31192 dup_rows_remote=22826 dedup_ratio=0.237976 router_tokens_per_expert=[886, 0, 0, 85, 111, 351, 638, 0, 0, 859, 173, 35, 203, 68, 104, 250, 12, 1948, 0, 13, 1018, 0, 4, 1100, 1874, 3, 0, 7534, 4806, 5980, 31, 533, 94, 339, 291, 1, 17, 716, 0, 5, 149, 1, 282, 1512, 0, 151, 3, 7867, 1, 13, 14, 0, 0, 98, 0, 6, 191, 0, 18, 215, 1, 4, 0, 52, 3252, 0, 2, 1, 467, 0, 7, 50, 682, 0, 987, 965, 107, 10, 194, 23, 0, 4, 0, 1, 78, 1, 1, 662, 1, 306, 9, 21, 58, 5, 54, 2, 10, 284, 2, 14, 34, 6, 50, 1, 245, 321, 402, 1348, 0, 95, 1798, 16, 7, 2765, 0, 15, 0, 94, 700, 5, 2, 15, 8015, 527, 7, 16, 0, 132] payload_rows_per_dst=[15340, 10256, 12698, 15192, 14740, 12672, 10942, 8040] dup_rows_per_dst=[6810, 2816, 4692, 1102, 8366, 1694, 3776, 1936] combine_rows_per_src=[15340, 10256, 12698, 15192, 14740, 12672, 10942, 8040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97808 payload_rows_remote=81614 dup_rows=33264 dup_rows_remote=26042 dedup_ratio=0.253784 router_tokens_per_expert=[810, 2, 0, 90, 124, 291, 595, 1, 3, 1134, 163, 41, 124, 32, 93, 156, 7, 853, 3, 8, 999, 11, 1, 1474, 2004, 8, 0, 7341, 4641, 6539, 5, 779, 68, 358, 179, 1, 5, 771, 2, 11, 138, 8, 650, 1778, 0, 63, 3, 7829, 14, 6, 27, 3, 0, 91, 0, 12, 241, 7, 40, 223, 6, 1, 3, 33, 2755, 10, 0, 1, 460, 0, 17, 22, 771, 0, 1147, 889, 76, 1, 143, 7, 0, 3, 0, 1, 37, 0, 6, 455, 1, 102, 26, 19, 14, 6, 137, 6, 31, 114, 9, 44, 27, 3, 41, 4, 228, 287, 386, 1437, 3, 79, 1547, 21, 4, 3064, 0, 8, 1, 119, 1111, 22, 0, 24, 8049, 771, 9, 13, 8, 57] payload_rows_per_dst=[16194, 14592, 12988, 14948, 9828, 13512, 7378, 8368] dup_rows_per_dst=[7222, 12474, 5260, 1148, 2770, 1192, 1144, 2054] combine_rows_per_src=[16194, 14592, 12988, 14948, 9828, 13512, 7378, 8368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=710164480 dispatch_meta_bytes=431816 combine_payload_bytes=710164480 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=710596296 ingress_bytes=936656896 link_bytes=936656896 token_bytes=1420760776 weight_movement_bytes=226492416 total_nvlink_bytes=1647253192 weight_to_token_ratio=0.159416 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.038603 gemm_active_groups=16 gemm_peak_group_rows=33664 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.187812789e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=629817344 dispatch_meta_bytes=413400 combine_payload_bytes=629817344 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=630230744 ingress_bytes=780812288 link_bytes=780812288 token_bytes=1260048088 weight_movement_bytes=150994944 total_nvlink_bytes=1411043032 weight_to_token_ratio=0.119833 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=25984 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.901798888e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=712605696 dispatch_meta_bytes=435096 combine_payload_bytes=712605696 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=713040792 ingress_bytes=1165590528 link_bytes=1165590528 token_bytes=1425646488 weight_movement_bytes=452984832 total_nvlink_bytes=1878631320 weight_to_token_ratio=0.317740 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.045956 gemm_active_groups=16 gemm_peak_group_rows=16384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478132859e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=688078848 dispatch_meta_bytes=455224 combine_payload_bytes=688078848 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=688534072 ingress_bytes=839073792 link_bytes=839073792 token_bytes=1376612920 weight_movement_bytes=150994944 total_nvlink_bytes=1527607864 weight_to_token_ratio=0.109686 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.036765 gemm_active_groups=19 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.064063677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=645709824 dispatch_meta_bytes=423112 combine_payload_bytes=645709824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=646132936 ingress_bytes=721207296 link_bytes=721207296 token_bytes=1291842760 weight_movement_bytes=75497472 total_nvlink_bytes=1367340232 weight_to_token_ratio=0.058442 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=25600 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.145923689e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=697466880 dispatch_meta_bytes=431864 combine_payload_bytes=697466880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=697898744 ingress_bytes=772964352 link_bytes=772964352 token_bytes=1395365624 weight_movement_bytes=75497472 total_nvlink_bytes=1470863096 weight_to_token_ratio=0.054106 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=21120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.802276013e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=96176 payload_rows_remote=82622 dup_rows=34896 dup_rows_remote=33342 dedup_ratio=0.266235 router_tokens_per_expert=[640, 1, 0, 62, 120, 249, 463, 1, 2, 649, 320, 16, 181, 76, 151, 128, 9, 1285, 0, 7, 736, 1, 0, 927, 1815, 15, 0, 7517, 5164, 6569, 2, 432, 228, 512, 185, 1, 24, 702, 0, 40, 87, 1, 404, 1403, 0, 163, 3, 7792, 0, 12, 16, 3, 0, 76, 4, 12, 38, 1, 20, 275, 17, 4, 5, 70, 2685, 1, 0, 0, 425, 0, 32, 169, 615, 0, 1092, 1611, 50, 17, 140, 10, 1, 5, 0, 22, 96, 1, 0, 712, 0, 76, 31, 5, 1, 3, 27, 5, 10, 93, 7, 19, 14, 11, 30, 12, 207, 479, 287, 1352, 6, 65, 2415, 3, 0, 3423, 0, 22, 0, 31, 1114, 3, 1, 44, 7967, 355, 8, 6, 4, 80] payload_rows_per_dst=[5380, 8524, 6426, 15216, 14928, 13554, 15898, 16250] dup_rows_per_dst=[738, 1934, 1080, 924, 9094, 1554, 9706, 9866] combine_rows_per_src=[5380, 8524, 6426, 15216, 14928, 13554, 15898, 16250] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=668581888 dispatch_meta_bytes=430624 combine_payload_bytes=668581888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=669012512 ingress_bytes=744079360 link_bytes=744079360 token_bytes=1337594400 weight_movement_bytes=75497472 total_nvlink_bytes=1413091872 weight_to_token_ratio=0.056443 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=39680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.435973656e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120002 payload_rows_remote=104154 dup_rows=11070 dup_rows_remote=11064 dedup_ratio=0.084457 router_tokens_per_expert=[0, 0, 0, 1, 0, 8164, 0, 0, 5, 0, 0, 0, 0, 4, 0, 0, 8177, 0, 0, 0, 21, 0, 0, 1, 6839, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 1, 0, 2, 0, 0, 3, 0, 0, 2, 0, 4269, 27, 0, 0, 0, 1, 64, 2, 0, 0, 7913, 0, 0, 12, 0, 11, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 3, 0, 0, 1, 8155, 0, 2, 0, 0, 14, 0, 8192, 0, 0, 19, 0, 3, 0, 17, 0, 2318, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3003, 0, 0, 4, 0, 0, 0, 0, 6, 8161, 0, 0, 0, 0, 1, 85, 0, 0, 1, 1] payload_rows_per_dst=[16338, 14058, 15848, 8694, 16384, 16334, 16008, 16338] dup_rows_per_dst=[10, 4478, 6, 60, 32, 88, 6224, 172] combine_rows_per_src=[16338, 14058, 15848, 8694, 16384, 16334, 16008, 16338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114752 payload_rows_remote=98434 dup_rows=16320 dup_rows_remote=12584 dedup_ratio=0.124512 router_tokens_per_expert=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 7, 0, 0, 221, 8069, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 8178, 0, 0, 0, 0, 0, 12, 0, 11, 0, 7846, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 2, 0, 57, 0, 8187, 0, 0, 6, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 93, 0, 1, 0, 0, 1, 0, 0, 0, 7317, 9, 1, 0, 0, 0, 936, 0, 0, 0, 0] payload_rows_per_dst=[16378, 870, 15696, 16368, 16360, 16378, 16384, 16318] dup_rows_per_dst=[0, 368, 0, 11866, 0, 152, 198, 3736] combine_rows_per_src=[16378, 870, 15696, 16368, 16360, 16378, 16384, 16318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=676839424 dispatch_meta_bytes=463856 combine_payload_bytes=676839424 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=677303280 ingress_bytes=827834368 link_bytes=827834368 token_bytes=1354142704 weight_movement_bytes=150994944 total_nvlink_bytes=1505137648 weight_to_token_ratio=0.111506 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2413 gemm_tail_pad_fraction=0.038603 gemm_active_groups=19 gemm_peak_group_rows=49536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.049810505e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122268 payload_rows_remote=105884 dup_rows=8804 dup_rows_remote=84 dedup_ratio=0.067169 router_tokens_per_expert=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4358, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7558, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 8, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4718, 0, 0, 2, 0, 0, 0, 0, 0, 7896, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16384, 16382, 15124, 16384, 16384, 9438, 15798] dup_rows_per_dst=[0, 8720, 32, 2, 0, 18, 32, 0] combine_rows_per_src=[16374, 16384, 16382, 15124, 16384, 16384, 9438, 15798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122126 payload_rows_remote=105826 dup_rows=8946 dup_rows_remote=8946 dedup_ratio=0.068253 router_tokens_per_expert=[16, 0, 0, 0, 0, 8181, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 7, 3873, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8148, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 2, 0, 8190, 0, 1, 1, 0, 0, 0, 0, 0, 108, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3856, 0, 0, 0, 0, 0, 0, 0, 0, 7987, 3, 0, 0, 0, 0, 607, 1, 0, 0, 0] payload_rows_per_dst=[16380, 7782, 16306, 16300, 16380, 16374, 16384, 16220] dup_rows_per_dst=[24, 2, 16, 0, 0, 2, 7926, 976] combine_rows_per_src=[16380, 7782, 16306, 16300, 16380, 16374, 16384, 16220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129366 payload_rows_remote=112984 dup_rows=1706 dup_rows_remote=1706 dedup_ratio=0.013016 router_tokens_per_expert=[0, 0, 0, 1, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 7691, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 1, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8051, 4, 0, 0, 0, 0, 2, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 6, 0, 8190, 0, 0, 0, 0, 0, 0, 2, 0, 12, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 654, 0, 0, 2, 0, 0, 0, 0, 0, 8067, 1, 0, 0, 1, 0, 122, 0, 0, 0, 0] payload_rows_per_dst=[16194, 15390, 16378, 16110, 16382, 16378, 16382, 16152] dup_rows_per_dst=[0, 10, 114, 4, 0, 16, 1332, 230] combine_rows_per_src=[16194, 15390, 16378, 16110, 16382, 16378, 16382, 16152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=853229568 dispatch_meta_bytes=460872 combine_payload_bytes=853229568 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=853690440 ingress_bytes=1004224512 link_bytes=1004224512 token_bytes=1706920008 weight_movement_bytes=150994944 total_nvlink_bytes=1857914952 weight_to_token_ratio=0.088460 gemm_rows_defined=67328 gemm_rows_charged=69632 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.033088 gemm_active_groups=16 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.273498037e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130554 payload_rows_remote=114170 dup_rows=518 dup_rows_remote=374 dedup_ratio=0.003952 router_tokens_per_expert=[0, 0, 0, 0, 0, 8185, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 8, 8040, 24, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8189, 0, 1, 5, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 29, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 12, 0, 0, 0, 0, 0, 8037, 1, 0, 0, 1, 0, 251, 0, 0, 0, 0] payload_rows_per_dst=[16372, 16100, 16282, 16374, 16384, 16382, 16384, 16276] dup_rows_per_dst=[2, 48, 2, 4, 4, 10, 144, 304] combine_rows_per_src=[16372, 16100, 16282, 16374, 16384, 16382, 16384, 16276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=806371328 dispatch_meta_bytes=444072 combine_payload_bytes=806371328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=806815400 ingress_bytes=881868800 link_bytes=881868800 token_bytes=1613186728 weight_movement_bytes=75497472 total_nvlink_bytes=1688684200 weight_to_token_ratio=0.046800 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=60416 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.118333771e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113674 payload_rows_remote=97380 dup_rows=17398 dup_rows_remote=17038 dedup_ratio=0.132736 router_tokens_per_expert=[0, 2, 0, 0, 0, 7976, 0, 0, 16, 15, 2, 0, 0, 0, 0, 0, 8187, 5, 0, 0, 7, 0, 0, 72, 7496, 0, 0, 0, 0, 0, 0, 7, 3, 1, 0, 0, 0, 17, 0, 12, 4, 6, 3, 2, 1, 6, 1, 0, 0, 20, 7776, 3, 2, 1, 0, 0, 0, 0, 0, 0, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 5, 1, 0, 9, 8161, 0, 1, 2, 0, 117, 0, 8170, 2, 10, 201, 1, 0, 0, 0, 4, 0, 7, 0, 0, 0, 0, 1, 0, 0, 12, 0, 3327, 0, 0, 3, 0, 0, 0, 0, 0, 5550, 0, 2, 4, 0, 3, 212, 1, 0, 0, 0] payload_rows_per_dst=[16294, 16380, 16340, 14294, 16360, 15852, 6698, 11456] dup_rows_per_dst=[360, 15494, 1234, 2, 12, 198, 10, 88] combine_rows_per_src=[16294, 16380, 16340, 14294, 16360, 15852, 6698, 11456] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=867401728 dispatch_meta_bytes=423872 combine_payload_bytes=867401728 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=867825600 ingress_bytes=1169391616 link_bytes=1169391616 token_bytes=1735227328 weight_movement_bytes=301989888 total_nvlink_bytes=2037217216 weight_to_token_ratio=0.174035 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=46080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.482953173e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=866926592 dispatch_meta_bytes=459088 combine_payload_bytes=866926592 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=867385680 ingress_bytes=1093419008 link_bytes=1093419008 token_bytes=1734312272 weight_movement_bytes=226492416 total_nvlink_bytes=1960804688 weight_to_token_ratio=0.130595 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=59520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.386609213e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=925564928 dispatch_meta_bytes=458760 combine_payload_bytes=925564928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=926023688 ingress_bytes=1001062400 link_bytes=1001062400 token_bytes=1851588616 weight_movement_bytes=75497472 total_nvlink_bytes=1927086088 weight_to_token_ratio=0.040774 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.038603 gemm_active_groups=13 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.269488033e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=935280640 dispatch_meta_bytes=458176 combine_payload_bytes=935280640 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935738816 ingress_bytes=1010778112 link_bytes=1010778112 token_bytes=1871019456 weight_movement_bytes=75497472 total_nvlink_bytes=1946516928 weight_to_token_ratio=0.040351 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=46848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281808924e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=797736960 dispatch_meta_bytes=457672 combine_payload_bytes=797736960 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=798194632 ingress_bytes=873234432 link_bytes=873234432 token_bytes=1595931592 weight_movement_bytes=75497472 total_nvlink_bytes=1671429064 weight_to_token_ratio=0.047306 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.107384177e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123756 payload_rows_remote=109244 dup_rows=7316 dup_rows_remote=4062 dedup_ratio=0.055817 router_tokens_per_expert=[8178, 0, 0, 0, 53, 0, 0, 1182, 0, 0, 0, 4596, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 8186, 0, 0, 0, 0, 7930, 0, 1, 0, 0, 0, 1715, 0, 7167, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 2, 178, 4, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 315, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 65, 1, 0, 0, 0, 5, 0, 1421, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 18, 51, 8190, 0, 0] payload_rows_per_dst=[11558, 16132, 14512, 16032, 16372, 16382, 16384, 16384] dup_rows_per_dst=[104, 0, 3254, 216, 2, 632, 2958, 150] combine_rows_per_src=[11558, 16132, 14512, 16032, 16372, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128878 payload_rows_remote=112494 dup_rows=2194 dup_rows_remote=2072 dedup_ratio=0.016739 router_tokens_per_expert=[0, 0, 0, 0, 0, 8171, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 9, 0, 0, 191, 7291, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 15, 0, 0, 0, 0, 8, 0, 0, 25, 0, 0, 0, 2, 7932, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8191, 0, 0, 4, 0, 8, 0, 8192, 0, 0, 46, 0, 0, 0, 4, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 546, 0, 0, 0, 0, 0, 0, 0, 0, 7390, 0, 0, 0, 0, 0, 1114, 0, 0, 0, 0] payload_rows_per_dst=[16344, 14892, 16360, 15868, 16384, 16384, 16380, 16266] dup_rows_per_dst=[2, 96, 76, 2, 6, 122, 1148, 742] combine_rows_per_src=[16344, 14892, 16360, 15868, 16384, 16384, 16380, 16266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 2, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128118 payload_rows_remote=111736 dup_rows=2954 dup_rows_remote=2872 dedup_ratio=0.022537 router_tokens_per_expert=[8189, 0, 0, 1, 1, 0, 0, 19, 0, 0, 0, 8047, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8189, 5, 0, 0, 7964, 0, 0, 0, 2, 8102, 2, 0, 0, 0, 0, 130, 0, 8180, 1, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 0, 0, 0, 3, 10, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 224, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 40] payload_rows_per_dst=[16104, 16380, 14326, 16208, 15950, 16384, 16384, 16382] dup_rows_per_dst=[36, 10, 248, 1030, 1030, 64, 454, 82] combine_rows_per_src=[16104, 16380, 14326, 16208, 15950, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127896 payload_rows_remote=111550 dup_rows=3176 dup_rows_remote=3176 dedup_ratio=0.024231 router_tokens_per_expert=[8189, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 6876, 23, 0, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 35, 0, 0, 7944, 37, 0, 0, 0, 8173, 0, 0, 0, 6, 0, 1335, 0, 8139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 122, 4, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 44, 0, 0, 43, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 22] payload_rows_per_dst=[13808, 16318, 16380, 16346, 15896, 16382, 16382, 16384] dup_rows_per_dst=[0, 48, 2654, 0, 2, 250, 88, 134] combine_rows_per_src=[13808, 16318, 16380, 16346, 15896, 16382, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123312 payload_rows_remote=107154 dup_rows=7760 dup_rows_remote=7760 dedup_ratio=0.059204 router_tokens_per_expert=[8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4585, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8079, 0, 0, 0, 8185, 8, 0, 0, 0, 8153, 0, 0, 0, 0, 0, 2145, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 802, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 816, 0, 0, 5, 0, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3] payload_rows_per_dst=[12358, 16158, 12968, 16306, 16370, 16384, 16384, 16384] dup_rows_per_dst=[1814, 0, 2646, 0, 34, 1606, 1632, 28] combine_rows_per_src=[12358, 16158, 12968, 16306, 16370, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=894926848 dispatch_meta_bytes=453224 combine_payload_bytes=894926848 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=895380072 ingress_bytes=1347911680 link_bytes=1347911680 token_bytes=1790306920 weight_movement_bytes=452984832 total_nvlink_bytes=2243291752 weight_to_token_ratio=0.253021 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=52736 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.709341743e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124344 payload_rows_remote=107960 dup_rows=6728 dup_rows_remote=6702 dedup_ratio=0.051331 router_tokens_per_expert=[8191, 0, 0, 0, 9, 0, 0, 121, 1, 1, 0, 4829, 3, 1, 0, 16, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 8189, 0, 0, 0, 3, 8160, 0, 0, 0, 1, 0, 2192, 0, 8175, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0, 0, 1, 0, 0, 6, 0, 0, 725, 0, 0, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 397, 0, 0, 0, 0, 2, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 5] payload_rows_per_dst=[9920, 16216, 16364, 16322, 16384, 16372, 16384, 16382] dup_rows_per_dst=[42, 0, 4378, 0, 26, 1454, 794, 34] combine_rows_per_src=[9920, 16216, 16364, 16322, 16384, 16372, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127030 payload_rows_remote=110654 dup_rows=4042 dup_rows_remote=4014 dedup_ratio=0.030838 router_tokens_per_expert=[8186, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6403, 59, 0, 0, 12, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 24, 0, 0, 8006, 0, 0, 0, 1, 8115, 0, 0, 0, 0, 0, 1700, 0, 8181, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 1, 4, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 255] payload_rows_per_dst=[12900, 16362, 16382, 16230, 16012, 16384, 16376, 16384] dup_rows_per_dst=[50, 38, 3384, 4, 4, 6, 28, 528] combine_rows_per_src=[12900, 16362, 16382, 16230, 16012, 16384, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109396 payload_rows_remote=93048 dup_rows=21676 dup_rows_remote=1228 dedup_ratio=0.165375 router_tokens_per_expert=[7965, 6, 0, 12, 142, 0, 0, 0, 0, 0, 0, 7177, 0, 0, 0, 52, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8150, 294, 0, 0, 8080, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 367, 0, 8099, 2, 2, 0, 0, 0, 0, 1, 0, 0, 13, 0, 0, 0, 0, 25, 2, 0, 4, 0, 1, 0, 0, 0, 163, 0, 6, 0, 18, 0, 0, 0, 0, 0, 225, 0, 0, 2, 0, 8162, 0, 1, 0, 0, 0, 0, 21, 45, 0, 1, 0, 0, 0, 3, 0, 7, 0, 10, 0, 0, 0, 0, 0, 126, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 13, 0, 15, 8175, 0, 7] payload_rows_per_dst=[16348, 16358, 7078, 16320, 16334, 11316, 10376, 15266] dup_rows_per_dst=[20448, 662, 52, 34, 210, 230, 34, 6] combine_rows_per_src=[16348, 16358, 7078, 16320, 16334, 11316, 10376, 15266] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=921550848 dispatch_meta_bytes=458264 combine_payload_bytes=921550848 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=922009112 ingress_bytes=1148043264 link_bytes=1148043264 token_bytes=1843559960 weight_movement_bytes=226492416 total_nvlink_bytes=2070052376 weight_to_token_ratio=0.122856 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.455880458e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=915341312 dispatch_meta_bytes=458432 combine_payload_bytes=915341312 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=915799744 ingress_bytes=1066336256 link_bytes=1066336256 token_bytes=1831141056 weight_movement_bytes=150994944 total_nvlink_bytes=1982136000 weight_to_token_ratio=0.082459 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.352264471e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=913817600 dispatch_meta_bytes=458904 combine_payload_bytes=913817600 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=914276504 ingress_bytes=1064812544 link_bytes=1064812544 token_bytes=1828094104 weight_movement_bytes=150994944 total_nvlink_bytes=1979089048 weight_to_token_ratio=0.082597 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.034926 gemm_active_groups=15 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.350332189e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=877805568 dispatch_meta_bytes=459656 combine_payload_bytes=877805568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=878265224 ingress_bytes=953303040 link_bytes=953303040 token_bytes=1756070792 weight_movement_bytes=75497472 total_nvlink_bytes=1831568264 weight_to_token_ratio=0.042992 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.208922442e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=884408320 dispatch_meta_bytes=458648 combine_payload_bytes=884408320 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=884866968 ingress_bytes=1035403264 link_bytes=1035403264 token_bytes=1769275288 weight_movement_bytes=150994944 total_nvlink_bytes=1920270232 weight_to_token_ratio=0.085343 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.313037083e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=906477568 dispatch_meta_bytes=458672 combine_payload_bytes=906477568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=906936240 ingress_bytes=981975040 link_bytes=981975040 token_bytes=1813413808 weight_movement_bytes=75497472 total_nvlink_bytes=1888911280 weight_to_token_ratio=0.041633 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=62464 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.245282574e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=762249216 dispatch_meta_bytes=377104 combine_payload_bytes=762249216 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=762626320 ingress_bytes=913244160 link_bytes=913244160 token_bytes=1524875536 weight_movement_bytes=150994944 total_nvlink_bytes=1675870480 weight_to_token_ratio=0.099021 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.044118 gemm_active_groups=14 gemm_peak_group_rows=50304 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.158122144e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128808 payload_rows_remote=114150 dup_rows=2264 dup_rows_remote=2264 dedup_ratio=0.017273 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8028, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 271, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8072, 8192, 7329, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 0, 915, 0, 0, 0, 0, 0, 0, 0, 0, 7468, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 6, 0, 0, 0, 0, 194, 0, 429, 0, 0] payload_rows_per_dst=[16056, 16384, 14658, 16380, 16196, 16384, 16384, 16366] dup_rows_per_dst=[0, 578, 0, 4, 570, 6, 6, 1100] combine_rows_per_src=[16056, 16384, 14658, 16380, 16196, 16384, 16384, 16366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114792 payload_rows_remote=98410 dup_rows=16280 dup_rows_remote=12560 dedup_ratio=0.124207 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 8186, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8191, 8188, 6298, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8187, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 0] payload_rows_per_dst=[16384, 15326, 1182, 16374, 16376, 16384, 16384, 16382] dup_rows_per_dst=[2, 42, 0, 0, 12442, 56, 18, 3720] combine_rows_per_src=[16384, 15326, 1182, 16374, 16376, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129862 payload_rows_remote=113478 dup_rows=1210 dup_rows_remote=368 dedup_ratio=0.009232 router_tokens_per_expert=[8192, 0, 0, 0, 3, 0, 0, 26, 0, 0, 0, 7756, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 3, 0, 0, 8164, 1, 0, 0, 0, 8012, 0, 0, 0, 0, 0, 30, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 418, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 75] payload_rows_per_dst=[15580, 16384, 16382, 16024, 16340, 16384, 16384, 16384] dup_rows_per_dst=[2, 6, 64, 0, 106, 842, 30, 160] combine_rows_per_src=[15580, 16384, 16382, 16024, 16340, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130920 payload_rows_remote=114540 dup_rows=152 dup_rows_remote=152 dedup_ratio=0.001160 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8187, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8189, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8190, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 1, 56, 0, 0] payload_rows_per_dst=[16380, 16384, 16344, 16380, 16286, 16380, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 10, 8, 0, 134] combine_rows_per_src=[16380, 16384, 16344, 16380, 16286, 16380, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118172 payload_rows_remote=101790 dup_rows=12900 dup_rows_remote=12900 dedup_ratio=0.098419 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8192, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 3, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16374, 16384, 16384, 3498] dup_rows_per_dst=[8, 0, 12888, 0, 0, 0, 2, 2] combine_rows_per_src=[16384, 16382, 16382, 16384, 16374, 16384, 16384, 3498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=935116800 dispatch_meta_bytes=465656 combine_payload_bytes=935116800 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=935582456 ingress_bytes=1312604160 link_bytes=1312604160 token_bytes=1870699256 weight_movement_bytes=377487360 total_nvlink_bytes=2248186616 weight_to_token_ratio=0.201789 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=50432 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.664566838e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131024 payload_rows_remote=114648 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 8093, 8186, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 3, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 110, 1, 0] payload_rows_per_dst=[16382, 16384, 16356, 16378, 16376, 16382, 16384, 16382] dup_rows_per_dst=[2, 4, 2, 0, 0, 2, 8, 30] combine_rows_per_src=[16382, 16384, 16356, 16378, 16376, 16382, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115508 payload_rows_remote=99202 dup_rows=15564 dup_rows_remote=15460 dedup_ratio=0.118744 router_tokens_per_expert=[0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 3, 5, 0, 3, 0, 0, 0, 0, 0, 32, 8158, 4, 0, 16, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2, 0, 1, 0, 8171, 8145, 6783, 0, 0, 1, 5, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 8151, 0, 0, 1, 0, 0, 0, 0, 0, 0, 11, 5827, 4, 0, 0, 0, 0, 0, 0, 0, 4, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 52, 24, 0, 0, 0, 0, 16, 7, 6, 0, 0, 15, 0, 0, 42, 0, 0, 0, 0, 1628, 0, 1995, 14, 4] payload_rows_per_dst=[16306, 16374, 16346, 16286, 11680, 16326, 16250, 5940] dup_rows_per_dst=[104, 294, 13580, 22, 6, 4, 86, 1468] combine_rows_per_src=[16306, 16374, 16346, 16286, 11680, 16326, 16250, 5940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=806174720 dispatch_meta_bytes=443880 combine_payload_bytes=806174720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=806618600 ingress_bytes=881672192 link_bytes=881672192 token_bytes=1612793320 weight_movement_bytes=75497472 total_nvlink_bytes=1688290792 weight_to_token_ratio=0.046812 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=50688 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.118084444e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125896 payload_rows_remote=109512 dup_rows=5176 dup_rows_remote=5162 dedup_ratio=0.039490 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 165, 3, 0, 0, 0, 0, 0, 8191, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 8186, 5949, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 2, 2402, 0, 0] payload_rows_per_dst=[16384, 16376, 11898, 16364, 15734, 16372, 16384, 16384] dup_rows_per_dst=[342, 8, 0, 2, 0, 0, 14, 4810] combine_rows_per_src=[16384, 16376, 11898, 16364, 15734, 16372, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=938311680 dispatch_meta_bytes=458768 combine_payload_bytes=938311680 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=938770448 ingress_bytes=1164804096 link_bytes=1164804096 token_bytes=1877082128 weight_movement_bytes=226492416 total_nvlink_bytes=2103574544 weight_to_token_ratio=0.120662 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.477135552e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=929611776 dispatch_meta_bytes=455384 combine_payload_bytes=929611776 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=930067160 ingress_bytes=1080606720 link_bytes=1080606720 token_bytes=1859678936 weight_movement_bytes=150994944 total_nvlink_bytes=2010673880 weight_to_token_ratio=0.081194 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=62336 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.370361428e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=833863680 dispatch_meta_bytes=458760 combine_payload_bytes=833863680 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=834322440 ingress_bytes=984858624 link_bytes=984858624 token_bytes=1668186120 weight_movement_bytes=150994944 total_nvlink_bytes=1819181064 weight_to_token_ratio=0.090514 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.248939365e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458784 combine_payload_bytes=939196416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939655200 ingress_bytes=1014693888 link_bytes=1014693888 token_bytes=1878851616 weight_movement_bytes=75497472 total_nvlink_bytes=1954349088 weight_to_token_ratio=0.040183 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=58368 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286774679e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=812662784 dispatch_meta_bytes=458648 combine_payload_bytes=812662784 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=813121432 ingress_bytes=888160256 link_bytes=888160256 token_bytes=1625784216 weight_movement_bytes=75497472 total_nvlink_bytes=1701281688 weight_to_token_ratio=0.046438 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.126312223e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=897122304 dispatch_meta_bytes=458696 combine_payload_bytes=897122304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=897581000 ingress_bytes=972619776 link_bytes=972619776 token_bytes=1794703304 weight_movement_bytes=75497472 total_nvlink_bytes=1870200776 weight_to_token_ratio=0.042067 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.233418783e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115054 payload_rows_remote=98674 dup_rows=16018 dup_rows_remote=15924 dedup_ratio=0.122208 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 40, 2, 0, 0, 8181, 0, 8040, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 103, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1339, 6479, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 20, 599, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16080, 16380, 16382, 1238, 16384, 16172, 16036] dup_rows_per_dst=[76, 0, 94, 15634, 0, 6, 208, 0] combine_rows_per_src=[16382, 16080, 16380, 16382, 1238, 16384, 16172, 16036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115348 payload_rows_remote=98986 dup_rows=15724 dup_rows_remote=13250 dedup_ratio=0.119965 router_tokens_per_expert=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 8188, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 85, 531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 1, 1, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 4, 0, 0, 0, 0] payload_rows_per_dst=[15968, 16272, 16384, 1226, 16384, 16382, 16370, 16362] dup_rows_per_dst=[0, 0, 10, 10, 13226, 0, 4, 2474] combine_rows_per_src=[15968, 16272, 16384, 1226, 16384, 16382, 16370, 16362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129706 payload_rows_remote=113322 dup_rows=1366 dup_rows_remote=24 dedup_ratio=0.010422 router_tokens_per_expert=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7722, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5, 670, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8185, 0, 0, 0, 0, 0, 0, 0, 7985, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 23, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 15444, 16380, 16384, 15970, 16382, 16378, 16384] dup_rows_per_dst=[0, 0, 8, 1342, 2, 4, 8, 2] combine_rows_per_src=[16384, 15444, 16380, 16384, 15970, 16382, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130970 payload_rows_remote=114616 dup_rows=102 dup_rows_remote=102 dedup_ratio=0.000778 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16354, 16382, 16384, 16320, 16384, 16384, 16378] dup_rows_per_dst=[0, 0, 12, 90, 0, 0, 0, 0] combine_rows_per_src=[16384, 16354, 16382, 16384, 16320, 16384, 16384, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=808337408 dispatch_meta_bytes=458392 combine_payload_bytes=808337408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=808795800 ingress_bytes=883834880 link_bytes=883834880 token_bytes=1617133208 weight_movement_bytes=75497472 total_nvlink_bytes=1692630680 weight_to_token_ratio=0.046686 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.120827037e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118532 payload_rows_remote=102148 dup_rows=12540 dup_rows_remote=12540 dedup_ratio=0.095673 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 8192, 5718, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 8192, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 4431, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 6078, 0, 0] payload_rows_per_dst=[16338, 16384, 11436, 16384, 8944, 16384, 16384, 16278] dup_rows_per_dst=[0, 152, 0, 114, 78, 0, 0, 12196] combine_rows_per_src=[16338, 16384, 11436, 16384, 8944, 16384, 16384, 16278] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130920 payload_rows_remote=114608 dup_rows=152 dup_rows_remote=142 dedup_ratio=0.001160 router_tokens_per_expert=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 5, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8179, 3, 0, 0, 0, 0, 1, 0, 8154, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 4, 1, 2, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 4, 1] payload_rows_per_dst=[16380, 16378, 16382, 16360, 16312, 16384, 16382, 16342] dup_rows_per_dst=[0, 0, 44, 68, 10, 0, 28, 2] combine_rows_per_src=[16380, 16378, 16382, 16360, 16312, 16384, 16382, 16342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126986 payload_rows_remote=110656 dup_rows=4086 dup_rows_remote=3702 dedup_ratio=0.031174 router_tokens_per_expert=[0, 0, 2, 0, 0, 8162, 0, 132, 0, 0, 0, 0, 0, 2, 8161, 0, 8170, 6, 0, 0, 21, 0, 2, 18, 26, 8035, 0, 0, 0, 0, 3, 0, 0, 0, 77, 0, 5, 1, 0, 2, 0, 9, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 463, 1, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8172, 3, 2, 14, 0, 1, 11, 6, 8007, 0, 20, 0, 1, 0, 0, 0, 0, 38, 0, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7723, 0, 0, 1, 29, 0] payload_rows_per_dst=[16330, 16384, 16142, 16370, 13608, 16342, 16308, 15502] dup_rows_per_dst=[384, 2684, 14, 926, 52, 20, 0, 6] combine_rows_per_src=[16330, 16384, 16142, 16370, 13608, 16342, 16308, 15502] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=810893312 dispatch_meta_bytes=448944 combine_payload_bytes=810893312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=811342256 ingress_bytes=886390784 link_bytes=886390784 token_bytes=1622235568 weight_movement_bytes=75497472 total_nvlink_bytes=1697733040 weight_to_token_ratio=0.046539 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.124068283e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118266 payload_rows_remote=102006 dup_rows=12806 dup_rows_remote=12732 dedup_ratio=0.097702 router_tokens_per_expert=[0, 1, 0, 0, 0, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 8191, 0, 7626, 0, 4, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 2, 0, 0, 0, 0, 10, 0, 0, 0, 118, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 2121, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 17, 8185, 0, 0, 0, 0, 1, 3, 0, 5667, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 1, 0, 0, 60, 0, 0, 63, 0, 0, 8042, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7934, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15260, 16358, 5372, 16382, 16382, 16260, 15868] dup_rows_per_dst=[8, 4, 134, 1204, 11364, 18, 74, 0] combine_rows_per_src=[16384, 15260, 16358, 5372, 16382, 16382, 16260, 15868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=928333824 dispatch_meta_bytes=453384 combine_payload_bytes=928333824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928787208 ingress_bytes=1003831296 link_bytes=1003831296 token_bytes=1857121032 weight_movement_bytes=75497472 total_nvlink_bytes=1932618504 weight_to_token_ratio=0.040653 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=48128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.272999383e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=938934272 dispatch_meta_bytes=458872 combine_payload_bytes=938934272 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939393144 ingress_bytes=1089929216 link_bytes=1089929216 token_bytes=1878327416 weight_movement_bytes=150994944 total_nvlink_bytes=2029322360 weight_to_token_ratio=0.080388 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382183665e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=836796416 dispatch_meta_bytes=458752 combine_payload_bytes=836796416 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=837255168 ingress_bytes=987791360 link_bytes=987791360 token_bytes=1674051584 weight_movement_bytes=150994944 total_nvlink_bytes=1825046528 weight_to_token_ratio=0.090197 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.252658487e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=938868736 dispatch_meta_bytes=459000 combine_payload_bytes=938868736 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939327736 ingress_bytes=1240858624 link_bytes=1240858624 token_bytes=1878196472 weight_movement_bytes=301989888 total_nvlink_bytes=2180186360 weight_to_token_ratio=0.160787 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.042279 gemm_active_groups=13 gemm_peak_group_rows=47872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.573583399e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=906493952 dispatch_meta_bytes=457432 combine_payload_bytes=906493952 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=906951384 ingress_bytes=1132986368 link_bytes=1132986368 token_bytes=1813445336 weight_movement_bytes=226492416 total_nvlink_bytes=2039937752 weight_to_token_ratio=0.124896 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.436786195e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=835633152 dispatch_meta_bytes=458952 combine_payload_bytes=835633152 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=836092104 ingress_bytes=911130624 link_bytes=911130624 token_bytes=1671725256 weight_movement_bytes=75497472 total_nvlink_bytes=1747222728 weight_to_token_ratio=0.045161 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.155441883e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121726 payload_rows_remote=105902 dup_rows=9346 dup_rows_remote=9346 dedup_ratio=0.071304 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8167, 189, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 105, 0, 0, 0, 0, 0, 0, 7807, 0, 0, 0, 0, 7931, 0, 1363, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 449, 17, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 3999, 0, 0, 6, 0, 0, 0, 0, 14, 0, 0, 0, 0, 2743, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16342, 16372, 15824, 16026, 16384, 8010, 16384, 16384] dup_rows_per_dst=[372, 2, 0, 2562, 928, 0, 5472, 10] combine_rows_per_src=[16342, 16372, 15824, 16026, 16384, 8010, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114626 payload_rows_remote=98262 dup_rows=16446 dup_rows_remote=16368 dedup_ratio=0.125473 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 1, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 8192, 0, 190, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 4969, 0, 0, 1, 0, 2986, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[6, 16376, 16372, 16384, 16382, 16360, 16382, 16364] dup_rows_per_dst=[4, 0, 0, 432, 20, 9938, 5974, 78] combine_rows_per_src=[6, 16376, 16372, 16384, 16382, 16360, 16382, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121894 payload_rows_remote=105530 dup_rows=9178 dup_rows_remote=8692 dedup_ratio=0.070023 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 467, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 0, 0, 8182, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 3704, 0, 0, 1, 0, 2520, 0, 0, 18, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1101, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16212, 16364, 16384, 7408, 16384, 16376] dup_rows_per_dst=[936, 0, 0, 486, 462, 2, 5094, 2198] combine_rows_per_src=[16382, 16384, 16212, 16364, 16384, 7408, 16384, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108440 payload_rows_remote=92056 dup_rows=22632 dup_rows_remote=22632 dedup_ratio=0.172668 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6743, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8185, 0, 138, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 816, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 8528, 16372, 16384, 10016, 7988, 16384] dup_rows_per_dst=[21342, 0, 0, 276, 872, 60, 82, 0] combine_rows_per_src=[16384, 16384, 8528, 16372, 16384, 10016, 7988, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=867549184 dispatch_meta_bytes=460992 combine_payload_bytes=867549184 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=868010176 ingress_bytes=1094041600 link_bytes=1094041600 token_bytes=1735559360 weight_movement_bytes=226492416 total_nvlink_bytes=1962051776 weight_to_token_ratio=0.130501 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=52864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.387398747e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120198 payload_rows_remote=103814 dup_rows=10874 dup_rows_remote=10874 dedup_ratio=0.082962 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 641, 3853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 4076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 10, 0, 0, 8162, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7858, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16212, 16346, 14476, 8296, 16384, 16384, 15716] dup_rows_per_dst=[8, 0, 2, 8844, 1908, 0, 112, 0] combine_rows_per_src=[16384, 16212, 16346, 14476, 8296, 16384, 16384, 15716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=90988 payload_rows_remote=74614 dup_rows=40084 dup_rows_remote=23776 dedup_ratio=0.305817 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 8154, 3, 0, 0, 6, 20, 0, 0, 8166, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 5, 0, 0, 0, 8159, 0, 14, 14, 0, 8170, 0, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 18, 22, 0, 0, 0, 0, 0, 8157, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 8153, 0, 0, 0, 12, 22, 4237, 0, 23, 0, 0, 5, 0, 0, 0, 9, 0, 0, 0, 518, 0, 0, 0, 0, 18, 0, 0, 10, 0, 0, 0, 0, 1, 1, 0, 0, 578, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[16374, 16374, 13498, 9382, 13874, 16368, 1066, 4052] dup_rows_per_dst=[16308, 24, 11656, 3512, 8, 8538, 34, 4] combine_rows_per_src=[16374, 16374, 13498, 9382, 13874, 16368, 1066, 4052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120052 payload_rows_remote=103668 dup_rows=11020 dup_rows_remote=10962 dedup_ratio=0.084076 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 8190, 4697, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 2, 0, 8114, 0, 0, 0, 0, 8180, 0, 118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 28, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 3, 1, 0, 0, 3260, 0, 1, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15366, 16384, 16232, 16372, 16384, 6560, 16382, 16372] dup_rows_per_dst=[9398, 0, 0, 226, 58, 1000, 270, 68] combine_rows_per_src=[15366, 16384, 16232, 16372, 16384, 6560, 16382, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114596 payload_rows_remote=98214 dup_rows=16476 dup_rows_remote=12484 dedup_ratio=0.125702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 81, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8182, 0, 1, 1, 0, 8101, 0, 33, 0, 2, 0, 0, 0, 5, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 65, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1217, 0, 0, 0, 1, 1988, 1, 1, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4921, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[162, 16384, 16368, 16212, 16382, 16326, 16382, 16380] dup_rows_per_dst=[0, 4, 2, 72, 130, 2434, 3992, 9842] combine_rows_per_src=[162, 16384, 16368, 16212, 16382, 16326, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=804962304 dispatch_meta_bytes=458520 combine_payload_bytes=804962304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=805420824 ingress_bytes=880459776 link_bytes=880459776 token_bytes=1610383128 weight_movement_bytes=75497472 total_nvlink_bytes=1685880600 weight_to_token_ratio=0.046882 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=58880 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116546930e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=864501760 dispatch_meta_bytes=456888 combine_payload_bytes=864501760 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=864958648 ingress_bytes=1015496704 link_bytes=1015496704 token_bytes=1729460408 weight_movement_bytes=150994944 total_nvlink_bytes=1880455352 weight_to_token_ratio=0.087308 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=59392 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287792763e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=754122752 dispatch_meta_bytes=458752 combine_payload_bytes=754122752 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=754581504 ingress_bytes=829620224 link_bytes=829620224 token_bytes=1508704256 weight_movement_bytes=75497472 total_nvlink_bytes=1584201728 weight_to_token_ratio=0.050041 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.052075222e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=850444288 dispatch_meta_bytes=458752 combine_payload_bytes=850444288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=850903040 ingress_bytes=925941760 link_bytes=925941760 token_bytes=1701347328 weight_movement_bytes=75497472 total_nvlink_bytes=1776844800 weight_to_token_ratio=0.044375 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.174224488e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=611237888 dispatch_meta_bytes=393560 combine_payload_bytes=611237888 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=611631448 ingress_bytes=762232832 link_bytes=762232832 token_bytes=1222869336 weight_movement_bytes=150994944 total_nvlink_bytes=1373864280 weight_to_token_ratio=0.123476 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=40448 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.666185234e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=849248256 dispatch_meta_bytes=458520 combine_payload_bytes=849248256 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=849706776 ingress_bytes=1000243200 link_bytes=1000243200 token_bytes=1698955032 weight_movement_bytes=150994944 total_nvlink_bytes=1849949976 weight_to_token_ratio=0.088875 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.268449172e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=804569088 dispatch_meta_bytes=442792 combine_payload_bytes=804569088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=805011880 ingress_bytes=880066560 link_bytes=880066560 token_bytes=1609580968 weight_movement_bytes=75497472 total_nvlink_bytes=1685078440 weight_to_token_ratio=0.046905 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=53120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116048277e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119890 payload_rows_remote=103506 dup_rows=11182 dup_rows_remote=8750 dedup_ratio=0.085312 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 401, 0, 0, 0, 0, 0, 0, 0, 198, 618, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7641, 8191, 0, 7216, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 814, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7938, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7653, 0, 0, 279, 0] payload_rows_per_dst=[16384, 8380, 16384, 16190, 14432, 16378, 15876, 15866] dup_rows_per_dst=[0, 8, 2432, 7104, 0, 1628, 0, 10] combine_rows_per_src=[16384, 8380, 16384, 16190, 14432, 16378, 15876, 15866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117432 payload_rows_remote=101050 dup_rows=13640 dup_rows_remote=12462 dedup_ratio=0.104065 router_tokens_per_expert=[1, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8167, 4, 2371, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 3, 0, 6, 11, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5751, 8190, 0, 8163, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 16, 0] payload_rows_per_dst=[16364, 4758, 16350, 16258, 14560, 16376, 16384, 16382] dup_rows_per_dst=[2, 6, 28, 9914, 8, 82, 2422, 1178] combine_rows_per_src=[16364, 4758, 16350, 16258, 14560, 16376, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128878 payload_rows_remote=112494 dup_rows=2194 dup_rows_remote=192 dedup_ratio=0.016739 router_tokens_per_expert=[0, 1, 19, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 8192, 0, 7153, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1000, 8192, 0, 8179, 0, 1, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0] payload_rows_per_dst=[16300, 14328, 16384, 16384, 16362, 16374, 16362, 16384] dup_rows_per_dst=[16, 0, 146, 2002, 0, 28, 0, 2] combine_rows_per_src=[16300, 14328, 16384, 16384, 16362, 16374, 16362, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=847921152 dispatch_meta_bytes=449024 combine_payload_bytes=847921152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=848370176 ingress_bytes=923418624 link_bytes=923418624 token_bytes=1696291328 weight_movement_bytes=75497472 total_nvlink_bytes=1771788800 weight_to_token_ratio=0.044507 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.171024796e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115760 payload_rows_remote=99376 dup_rows=15312 dup_rows_remote=2714 dedup_ratio=0.116821 router_tokens_per_expert=[0, 0, 12, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 6299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 560, 8192, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 1120, 16380, 16384, 16340, 16384] dup_rows_per_dst=[24, 12598, 2686, 0, 0, 4, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 1120, 16380, 16384, 16340, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104734 payload_rows_remote=88956 dup_rows=26338 dup_rows_remote=25486 dedup_ratio=0.200943 router_tokens_per_expert=[0, 0, 0, 0, 47, 6712, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 25, 0, 0, 0, 40, 4, 8137, 0, 8048, 0, 14, 0, 0, 26, 0, 0, 0, 0, 6, 0, 3, 0, 0, 0, 0, 0, 685, 0, 0, 0, 0, 0, 0, 39, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8123, 0, 8171, 0, 0, 0, 0, 8135, 12, 1, 28, 0, 0, 0, 0, 0, 4, 15, 0, 0, 0, 0, 0, 0, 0, 0, 853, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 312, 0, 7869, 0, 0, 0, 2, 16, 0, 0, 0, 0, 0, 0, 20, 8162, 0, 0, 0, 0] payload_rows_per_dst=[15778, 16384, 10140, 114, 16380, 13210, 16346, 16382] dup_rows_per_dst=[852, 20342, 246, 0, 4848, 0, 32, 18] combine_rows_per_src=[15778, 16384, 10140, 114, 16380, 13210, 16346, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116358 payload_rows_remote=99974 dup_rows=14714 dup_rows_remote=3046 dedup_ratio=0.112259 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 848, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8181, 0, 15, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5830, 0, 0, 4, 0, 1392, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 88, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1696, 16384, 16380, 16362, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 32, 36, 11668, 2802, 176] combine_rows_per_src=[1696, 16384, 16380, 16362, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130440 payload_rows_remote=114058 dup_rows=632 dup_rows_remote=632 dedup_ratio=0.004822 router_tokens_per_expert=[0, 2, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 44, 0, 8191, 0, 8057, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 132, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 108, 8190, 0, 8191, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8113, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0] payload_rows_per_dst=[16196, 16116, 16384, 16382, 16382, 16380, 16236, 16364] dup_rows_per_dst=[0, 90, 268, 254, 0, 20, 0, 0] combine_rows_per_src=[16196, 16116, 16384, 16382, 16382, 16380, 16236, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127160 payload_rows_remote=112470 dup_rows=3912 dup_rows_remote=3912 dedup_ratio=0.029846 router_tokens_per_expert=[0, 2, 0, 0, 1, 8182, 0, 0, 0, 0, 20, 0, 0, 0, 0, 3, 0, 0, 0, 10, 0, 0, 7, 0, 8189, 0, 7811, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 88, 74, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1052, 8191, 0, 7759, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 703, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7, 0, 7337, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7903, 1, 0, 0, 0] payload_rows_per_dst=[16370, 15626, 16380, 16384, 15520, 16382, 14690, 15808] dup_rows_per_dst=[46, 30, 324, 2104, 0, 1406, 0, 2] combine_rows_per_src=[16370, 15626, 16380, 16384, 15520, 16382, 14690, 15808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=827801600 dispatch_meta_bytes=454048 combine_payload_bytes=827801600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=828255648 ingress_bytes=903299072 link_bytes=903299072 token_bytes=1656057248 weight_movement_bytes=75497472 total_nvlink_bytes=1731554720 weight_to_token_ratio=0.045589 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.145510372e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=921550848 dispatch_meta_bytes=450744 combine_payload_bytes=921550848 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=922001592 ingress_bytes=1223540736 link_bytes=1223540736 token_bytes=1843552440 weight_movement_bytes=301989888 total_nvlink_bytes=2145542328 weight_to_token_ratio=0.163809 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=44160 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.551621880e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=814088192 dispatch_meta_bytes=408360 combine_payload_bytes=814088192 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=814496552 ingress_bytes=965083136 link_bytes=965083136 token_bytes=1628584744 weight_movement_bytes=150994944 total_nvlink_bytes=1779579688 weight_to_token_ratio=0.092715 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=42880 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.223861262e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=728727552 dispatch_meta_bytes=457768 combine_payload_bytes=728727552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=729185320 ingress_bytes=804225024 link_bytes=804225024 token_bytes=1457912872 weight_movement_bytes=75497472 total_nvlink_bytes=1533410344 weight_to_token_ratio=0.051785 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.019870534e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=818987008 dispatch_meta_bytes=412080 combine_payload_bytes=818987008 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=819399088 ingress_bytes=969981952 link_bytes=969981952 token_bytes=1638386096 weight_movement_bytes=150994944 total_nvlink_bytes=1789381040 weight_to_token_ratio=0.092161 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=28032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.230073650e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=934363136 dispatch_meta_bytes=458760 combine_payload_bytes=934363136 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=934821896 ingress_bytes=1160855552 link_bytes=1160855552 token_bytes=1869185032 weight_movement_bytes=226492416 total_nvlink_bytes=2095677448 weight_to_token_ratio=0.121172 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=62848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.472128243e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=921354240 dispatch_meta_bytes=465528 combine_payload_bytes=921354240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=921819768 ingress_bytes=996851712 link_bytes=996851712 token_bytes=1843174008 weight_movement_bytes=75497472 total_nvlink_bytes=1918671480 weight_to_token_ratio=0.040961 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=64000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.264148288e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126094 payload_rows_remote=109710 dup_rows=4978 dup_rows_remote=4978 dedup_ratio=0.037979 router_tokens_per_expert=[0, 0, 0, 0, 0, 50, 0, 0, 4, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 5933, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2769, 0, 0, 7037, 8192, 0, 0, 14, 0, 0, 0, 0, 1, 0, 0, 0, 0, 855, 0, 7212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 8122, 15, 0, 0, 4, 0, 0, 0, 0, 0, 724, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 11882, 16382, 15968, 16384, 16326] dup_rows_per_dst=[114, 12, 0, 0, 3258, 168, 4, 1422] combine_rows_per_src=[16384, 16384, 16384, 11882, 16382, 15968, 16384, 16326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129786 payload_rows_remote=113404 dup_rows=1286 dup_rows_remote=548 dedup_ratio=0.009811 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 27, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8072, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8035, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 165, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8191, 2, 0, 5, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 8, 0] payload_rows_per_dst=[15688, 16384, 16382, 16152, 16032, 16384, 16382, 16382] dup_rows_per_dst=[4, 54, 0, 2, 0, 330, 158, 738] combine_rows_per_src=[15688, 16384, 16382, 16152, 16032, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=898744320 dispatch_meta_bytes=458752 combine_payload_bytes=898744320 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=899203072 ingress_bytes=1049739264 link_bytes=1049739264 token_bytes=1797947392 weight_movement_bytes=150994944 total_nvlink_bytes=1948942336 weight_to_token_ratio=0.083982 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.331217149e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128960 payload_rows_remote=114656 dup_rows=2112 dup_rows_remote=2112 dedup_ratio=0.016113 router_tokens_per_expert=[0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 322, 0, 0, 8107, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 132, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8177, 12, 0, 1, 0, 2, 0, 0, 26, 0, 645, 3, 0] payload_rows_per_dst=[16384, 16384, 16384, 14304, 16372, 16374, 16382, 16376] dup_rows_per_dst=[6, 2, 0, 0, 486, 262, 0, 1356] combine_rows_per_src=[16384, 16384, 16384, 14304, 16372, 16374, 16382, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128388 payload_rows_remote=112004 dup_rows=2684 dup_rows_remote=2684 dedup_ratio=0.020477 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2135, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3, 0, 47, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 14602, 16380, 15486, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 842, 0, 1742, 0, 100] combine_rows_per_src=[16384, 16384, 16384, 14602, 16380, 15486, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116208 payload_rows_remote=99824 dup_rows=14864 dup_rows_remote=14274 dedup_ratio=0.113403 router_tokens_per_expert=[0, 29, 12, 0, 0, 0, 0, 0, 418, 0, 0, 4, 89, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 33, 0, 6, 0, 0, 0, 0, 2, 30, 0, 0, 0, 0, 0, 0, 5156, 0, 0, 0, 1, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 7698, 7935, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 8091, 0, 0, 142, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 19, 8054, 0, 0, 2, 8143, 0, 47, 0, 0, 0, 8143, 13, 0, 0, 0, 0, 0, 0, 19, 33, 1152, 135, 62] payload_rows_per_dst=[16384, 16192, 16352, 16248, 15428, 7648, 16228, 11728] dup_rows_per_dst=[590, 10, 34, 10312, 2, 3512, 12, 392] combine_rows_per_src=[16384, 16192, 16352, 16248, 15428, 7648, 16228, 11728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120396 payload_rows_remote=104012 dup_rows=10676 dup_rows_remote=10640 dedup_ratio=0.081451 router_tokens_per_expert=[0, 0, 2, 0, 0, 8172, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 54, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5239, 8192, 0, 8058, 0, 0, 0, 42, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0] payload_rows_per_dst=[16346, 6022, 16384, 16384, 16200, 16384, 16384, 16292] dup_rows_per_dst=[10, 0, 152, 10478, 0, 36, 0, 0] combine_rows_per_src=[16346, 6022, 16384, 16384, 16200, 16384, 16384, 16292] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=929005568 dispatch_meta_bytes=455808 combine_payload_bytes=929005568 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=929461376 ingress_bytes=1080000512 link_bytes=1080000512 token_bytes=1858466944 weight_movement_bytes=150994944 total_nvlink_bytes=2009461888 weight_to_token_ratio=0.081247 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=61952 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.369592671e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130514 payload_rows_remote=114134 dup_rows=558 dup_rows_remote=558 dedup_ratio=0.004257 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 214, 1, 0, 7850, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 8119, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0] payload_rows_per_dst=[16382, 16380, 16384, 16186, 16094, 16326, 16380, 16382] dup_rows_per_dst=[0, 4, 4, 0, 36, 0, 0, 514] combine_rows_per_src=[16382, 16380, 16384, 16186, 16094, 16326, 16380, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126730 payload_rows_remote=112586 dup_rows=4342 dup_rows_remote=4334 dedup_ratio=0.033127 router_tokens_per_expert=[0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 0, 0, 8185, 0, 0, 48, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 7165, 0, 0, 0, 3, 0, 0, 0, 0, 2, 12, 0, 0, 54, 0, 0, 7008, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2047, 0, 8176, 0, 0, 10, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8184, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 1, 0, 0, 3, 44, 11, 0] payload_rows_per_dst=[16354, 16384, 16380, 14348, 14144, 16376, 16372, 16372] dup_rows_per_dst=[14, 114, 0, 4, 8, 4092, 2, 108] combine_rows_per_src=[16354, 16384, 16380, 14348, 14144, 16376, 16372, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939261952 dispatch_meta_bytes=467072 combine_payload_bytes=939261952 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939729024 ingress_bytes=1014759424 link_bytes=1014759424 token_bytes=1878990976 weight_movement_bytes=75497472 total_nvlink_bytes=1954488448 weight_to_token_ratio=0.040180 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=55680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286857788e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=917536768 dispatch_meta_bytes=458752 combine_payload_bytes=917536768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=917995520 ingress_bytes=993034240 link_bytes=993034240 token_bytes=1835532288 weight_movement_bytes=75497472 total_nvlink_bytes=1911029760 weight_to_token_ratio=0.041131 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.259307196e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=817758208 dispatch_meta_bytes=456392 combine_payload_bytes=817758208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=818214600 ingress_bytes=893255680 link_bytes=893255680 token_bytes=1635972808 weight_movement_bytes=75497472 total_nvlink_bytes=1711470280 weight_to_token_ratio=0.046148 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.132773937e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=852066304 dispatch_meta_bytes=458608 combine_payload_bytes=852066304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=852524912 ingress_bytes=927563776 link_bytes=927563776 token_bytes=1704591216 weight_movement_bytes=75497472 total_nvlink_bytes=1780088688 weight_to_token_ratio=0.044291 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63104 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176281432e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=934985728 dispatch_meta_bytes=458768 combine_payload_bytes=934985728 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=935444496 ingress_bytes=1161478144 link_bytes=1161478144 token_bytes=1870430224 weight_movement_bytes=226492416 total_nvlink_bytes=2096922640 weight_to_token_ratio=0.121091 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.472917777e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=922304512 dispatch_meta_bytes=467680 combine_payload_bytes=922304512 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=922772192 ingress_bytes=1224294400 link_bytes=1224294400 token_bytes=1845076704 weight_movement_bytes=301989888 total_nvlink_bytes=2147066592 weight_to_token_ratio=0.163673 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=61440 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.552577632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129748 payload_rows_remote=113364 dup_rows=1324 dup_rows_remote=1048 dedup_ratio=0.010101 router_tokens_per_expert=[0, 71, 0, 8189, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 139, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7788, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7950, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 430, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 15580, 16342, 15910, 16384, 16382] dup_rows_per_dst=[142, 0, 276, 0, 0, 4, 880, 22] combine_rows_per_src=[16382, 16384, 16384, 15580, 16342, 15910, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127766 payload_rows_remote=111382 dup_rows=3306 dup_rows_remote=3300 dedup_ratio=0.025223 router_tokens_per_expert=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8190, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8188, 33, 0, 0, 0, 0, 0, 0, 15, 10, 0, 0, 0, 0, 8153, 0, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 8188, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14490, 16382, 15062, 16308, 16384, 16380, 16376, 16384] dup_rows_per_dst=[8, 1880, 42, 0, 134, 1236, 0, 6] combine_rows_per_src=[14490, 16382, 15062, 16308, 16384, 16380, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=928677888 dispatch_meta_bytes=457648 combine_payload_bytes=928677888 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=929135536 ingress_bytes=1230667776 link_bytes=1230667776 token_bytes=1857813424 weight_movement_bytes=301989888 total_nvlink_bytes=2159803312 weight_to_token_ratio=0.162551 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.042279 gemm_active_groups=13 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.560659970e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130674 payload_rows_remote=114608 dup_rows=398 dup_rows_remote=398 dedup_ratio=0.003036 router_tokens_per_expert=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 47, 0, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 1, 0, 0, 0, 3, 0, 0, 8183, 0, 8033, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 45, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16308, 16382, 16066, 16384, 16384, 16384, 16382] dup_rows_per_dst=[94, 0, 4, 0, 0, 298, 0, 2] combine_rows_per_src=[16384, 16308, 16382, 16066, 16384, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131020 payload_rows_remote=114636 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16336, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 4, 48, 0] combine_rows_per_src=[16380, 16384, 16384, 16336, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125834 payload_rows_remote=109454 dup_rows=5238 dup_rows_remote=4864 dedup_ratio=0.039963 router_tokens_per_expert=[23, 0, 0, 8141, 0, 0, 0, 0, 0, 4, 7423, 860, 12, 0, 0, 0, 0, 0, 0, 2, 7816, 0, 0, 0, 0, 0, 0, 29, 1, 0, 0, 1, 8098, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 10, 8046, 0, 8117, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 28, 40, 0, 0, 0, 5, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 277, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 102, 0, 0, 16, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 1, 0, 0, 0, 0, 10, 0, 25, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 12, 0] payload_rows_per_dst=[16380, 15690, 16266, 16384, 16298, 13266, 15208, 16342] dup_rows_per_dst=[374, 8, 1022, 3588, 0, 204, 26, 16] combine_rows_per_src=[16380, 15690, 16266, 16384, 16298, 13266, 15208, 16342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129624 payload_rows_remote=113240 dup_rows=1448 dup_rows_remote=1418 dedup_ratio=0.011047 router_tokens_per_expert=[0, 0, 40, 8189, 0, 0, 0, 0, 2, 0, 8191, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 7502, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16318, 16382, 15012, 16384, 16380, 16384, 16380] dup_rows_per_dst=[104, 0, 0, 2, 0, 1312, 30, 0] combine_rows_per_src=[16384, 16318, 16382, 15012, 16384, 16380, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=912441344 dispatch_meta_bytes=458728 combine_payload_bytes=912441344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=912900072 ingress_bytes=987938816 link_bytes=987938816 token_bytes=1825341416 weight_movement_bytes=75497472 total_nvlink_bytes=1900838888 weight_to_token_ratio=0.041361 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.252845482e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129718 payload_rows_remote=113340 dup_rows=1354 dup_rows_remote=1354 dedup_ratio=0.010330 router_tokens_per_expert=[0, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 8189, 584, 0, 0, 0, 1, 0, 0, 0, 0, 7968, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8172, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 87, 0, 0, 8155, 0, 7748, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0, 0, 0, 4, 0, 0, 0, 0, 15, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 37, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 15938, 16382, 15498, 16378, 16384, 16382, 16374] dup_rows_per_dst=[1168, 0, 106, 2, 0, 40, 38, 0] combine_rows_per_src=[16382, 15938, 16382, 15498, 16378, 16384, 16382, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130836 payload_rows_remote=114452 dup_rows=236 dup_rows_remote=216 dedup_ratio=0.001801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8095, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 0, 0, 7793, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16190, 16344, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 106, 20, 0, 110] combine_rows_per_src=[16384, 16384, 16384, 16190, 16344, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938868736 dispatch_meta_bytes=460024 combine_payload_bytes=938868736 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939328760 ingress_bytes=1014366208 link_bytes=1014366208 token_bytes=1878197496 weight_movement_bytes=75497472 total_nvlink_bytes=1953694968 weight_to_token_ratio=0.040197 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=63744 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286359135e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=939098112 dispatch_meta_bytes=458752 combine_payload_bytes=939098112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939556864 ingress_bytes=1014595584 link_bytes=1014595584 token_bytes=1878654976 weight_movement_bytes=75497472 total_nvlink_bytes=1954152448 weight_to_token_ratio=0.040187 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286650016e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=896647168 dispatch_meta_bytes=457272 combine_payload_bytes=896647168 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=897104440 ingress_bytes=1123139584 link_bytes=1123139584 token_bytes=1793751608 weight_movement_bytes=226492416 total_nvlink_bytes=2020244024 weight_to_token_ratio=0.126267 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.424299086e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=927662080 dispatch_meta_bytes=458632 combine_payload_bytes=927662080 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928120712 ingress_bytes=1003159552 link_bytes=1003159552 token_bytes=1855782792 weight_movement_bytes=75497472 total_nvlink_bytes=1931280264 weight_to_token_ratio=0.040682 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.272147517e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118618 payload_rows_remote=102234 dup_rows=12454 dup_rows_remote=12404 dedup_ratio=0.095016 router_tokens_per_expert=[0, 0, 0, 3828, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5738, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7127, 0, 687, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 8187, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2968, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[7660, 16384, 16384, 15694, 16380, 16266, 13466, 16384] dup_rows_per_dst=[0, 11476, 50, 10, 4, 12, 830, 72] combine_rows_per_src=[7660, 16384, 16384, 15694, 16380, 16266, 13466, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=928481280 dispatch_meta_bytes=458776 combine_payload_bytes=928481280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928940056 ingress_bytes=1003978752 link_bytes=1003978752 token_bytes=1857421336 weight_movement_bytes=75497472 total_nvlink_bytes=1932918808 weight_to_token_ratio=0.040646 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.273186378e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=937590784 dispatch_meta_bytes=458672 combine_payload_bytes=937590784 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938049456 ingress_bytes=1088585728 link_bytes=1088585728 token_bytes=1875640240 weight_movement_bytes=150994944 total_nvlink_bytes=2026635184 weight_to_token_ratio=0.080503 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=54656 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.380479933e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=837500928 dispatch_meta_bytes=458552 combine_payload_bytes=837500928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=837959480 ingress_bytes=912998400 link_bytes=912998400 token_bytes=1675460408 weight_movement_bytes=75497472 total_nvlink_bytes=1750957880 weight_to_token_ratio=0.045061 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.157810485e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108852 payload_rows_remote=92476 dup_rows=22220 dup_rows_remote=21912 dedup_ratio=0.169525 router_tokens_per_expert=[0, 0, 0, 7617, 244, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 102, 25, 0, 0, 0, 0, 0, 0, 8022, 0, 262, 1, 0, 0, 0, 0, 8183, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 1, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5748, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 2622, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] payload_rows_per_dst=[15678, 1836, 15598, 16380, 16384, 10268, 16332, 16376] dup_rows_per_dst=[46, 0, 250, 14730, 110, 0, 6776, 308] combine_rows_per_src=[15678, 1836, 15598, 16380, 16384, 10268, 16332, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128148 payload_rows_remote=111854 dup_rows=2924 dup_rows_remote=2924 dedup_ratio=0.022308 router_tokens_per_expert=[0, 0, 0, 6713, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 6, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4762, 0, 5, 11, 0, 0, 0, 0, 0, 0, 0, 4869, 0, 0, 37, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8041] payload_rows_per_dst=[13788, 16380, 16378, 16294, 16384, 16380, 16384, 16160] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 2910, 6] combine_rows_per_src=[13788, 16380, 16378, 16294, 16384, 16380, 16384, 16160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115424 payload_rows_remote=99040 dup_rows=15648 dup_rows_remote=14206 dedup_ratio=0.119385 router_tokens_per_expert=[0, 0, 0, 7184, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2884, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 5566, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16186, 16384, 16384, 6170, 16384, 16382, 11150, 16384] dup_rows_per_dst=[12282, 1442, 0, 1874, 2, 0, 42, 6] combine_rows_per_src=[16186, 16384, 16384, 6170, 16384, 16382, 11150, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129398 payload_rows_remote=113254 dup_rows=1674 dup_rows_remote=702 dedup_ratio=0.012772 router_tokens_per_expert=[0, 0, 0, 7308, 415, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 6, 0, 1, 0, 0, 0, 0, 7945, 0, 225, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 5, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3812, 0, 6, 278, 0, 0, 2, 0, 0, 0, 0, 4460, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158] payload_rows_per_dst=[15452, 16370, 16382, 15906, 16384, 16384, 16144, 16376] dup_rows_per_dst=[0, 248, 4, 434, 2, 12, 972, 2] combine_rows_per_src=[15452, 16370, 16382, 15906, 16384, 16384, 16144, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118638 payload_rows_remote=102612 dup_rows=12434 dup_rows_remote=6932 dedup_ratio=0.094864 router_tokens_per_expert=[22, 0, 0, 2748, 13, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 9, 0, 0, 0, 0, 17, 6, 0, 0, 5, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 227, 0, 0, 0, 2, 0, 27, 7944, 0, 0, 50, 0, 0, 0, 0, 7786, 0, 0, 1, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 0, 0, 0, 0, 3066, 0, 26, 0, 8143, 34, 0, 0, 0, 0, 0, 12, 0, 2, 0, 0, 0, 0, 12, 5489, 27, 9, 0, 0, 57, 0, 0, 35, 8, 25, 5253, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4, 0, 8104] payload_rows_per_dst=[16026, 15706, 16314, 11078, 16268, 16384, 10644, 16218] dup_rows_per_dst=[5502, 22, 434, 56, 0, 6182, 238, 0] combine_rows_per_src=[16026, 15706, 16314, 11078, 16268, 16384, 10644, 16218] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=757563392 dispatch_meta_bytes=457552 combine_payload_bytes=757563392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=758020944 ingress_bytes=833060864 link_bytes=833060864 token_bytes=1515584336 weight_movement_bytes=75497472 total_nvlink_bytes=1591081808 weight_to_token_ratio=0.049814 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.056438438e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=916307968 dispatch_meta_bytes=459112 combine_payload_bytes=916307968 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=916767080 ingress_bytes=1142800384 link_bytes=1142800384 token_bytes=1833075048 weight_movement_bytes=226492416 total_nvlink_bytes=2059567464 weight_to_token_ratio=0.123559 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.049632 gemm_active_groups=10 gemm_peak_group_rows=48384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.449231748e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127622 payload_rows_remote=111244 dup_rows=3450 dup_rows_remote=3450 dedup_ratio=0.026321 router_tokens_per_expert=[0, 0, 0, 6632, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 1, 0, 8033, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 11, 0, 1, 8188, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4140, 2, 341, 0, 0, 0, 4, 0, 0, 0, 0, 5400, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8182] payload_rows_per_dst=[13264, 16082, 16382, 16382, 16378, 16384, 16382, 16368] dup_rows_per_dst=[0, 0, 2, 2, 0, 26, 3392, 28] combine_rows_per_src=[13264, 16082, 16382, 16382, 16378, 16384, 16382, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=811335680 dispatch_meta_bytes=452984 combine_payload_bytes=811335680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=811788664 ingress_bytes=886833152 link_bytes=886833152 token_bytes=1623124344 weight_movement_bytes=75497472 total_nvlink_bytes=1698621816 weight_to_token_ratio=0.046514 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=57728 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.124629268e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130364 payload_rows_remote=113990 dup_rows=708 dup_rows_remote=196 dedup_ratio=0.005402 router_tokens_per_expert=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 4, 0, 4, 0, 0, 0, 0, 0, 0, 7890, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 8142, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 250, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8186, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15780, 16380, 16296, 16384, 16374, 16384, 16382] dup_rows_per_dst=[16, 0, 0, 0, 0, 512, 164, 16] combine_rows_per_src=[16384, 15780, 16380, 16296, 16384, 16374, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=927776768 dispatch_meta_bytes=455824 combine_payload_bytes=927776768 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=928232592 ingress_bytes=1078771712 link_bytes=1078771712 token_bytes=1856009360 weight_movement_bytes=150994944 total_nvlink_bytes=2007004304 weight_to_token_ratio=0.081355 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.044118 gemm_active_groups=14 gemm_peak_group_rows=31616 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.368034379e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130274 payload_rows_remote=113892 dup_rows=798 dup_rows_remote=114 dedup_ratio=0.006088 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7943, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 288, 56, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 8181, 0, 0, 0, 8189, 0, 0, 0, 49, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15920, 16382, 16384, 16256, 16180, 16384, 16384] dup_rows_per_dst=[2, 0, 684, 18, 0, 0, 94, 0] combine_rows_per_src=[16384, 15920, 16382, 16384, 16256, 16180, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=840597504 dispatch_meta_bytes=438176 combine_payload_bytes=840597504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=841035680 ingress_bytes=916094976 link_bytes=916094976 token_bytes=1681633184 weight_movement_bytes=75497472 total_nvlink_bytes=1757130656 weight_to_token_ratio=0.044895 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=49664 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.161737380e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=911310848 dispatch_meta_bytes=458776 combine_payload_bytes=911310848 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=911769624 ingress_bytes=1137803264 link_bytes=1137803264 token_bytes=1823080472 weight_movement_bytes=226492416 total_nvlink_bytes=2049572888 weight_to_token_ratio=0.124236 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.442894697e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=933806080 dispatch_meta_bytes=456744 combine_payload_bytes=933806080 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=934262824 ingress_bytes=1084801024 link_bytes=1084801024 token_bytes=1868068904 weight_movement_bytes=150994944 total_nvlink_bytes=2019063848 weight_to_token_ratio=0.080829 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=63488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.375680396e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=933003264 dispatch_meta_bytes=456024 combine_payload_bytes=933003264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933459288 ingress_bytes=1008500736 link_bytes=1008500736 token_bytes=1866462552 weight_movement_bytes=75497472 total_nvlink_bytes=1941960024 weight_to_token_ratio=0.040449 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278920890e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128562 payload_rows_remote=112246 dup_rows=2510 dup_rows_remote=1842 dedup_ratio=0.019150 router_tokens_per_expert=[0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 17, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 8186, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16050, 16378, 14310, 16384, 16384, 16358, 16382, 16316] dup_rows_per_dst=[24, 0, 40, 16, 356, 1330, 76, 668] combine_rows_per_src=[16050, 16378, 14310, 16384, 16384, 16358, 16382, 16316] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130968 payload_rows_remote=114584 dup_rows=104 dup_rows_remote=102 dedup_ratio=0.000793 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8185, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 8191, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16302, 16374, 16384] dup_rows_per_dst=[2, 0, 92, 2, 0, 0, 8, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16302, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114676 dup_rows=12 dup_rows_remote=6 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16376, 16382, 16384, 16384] dup_rows_per_dst=[0, 6, 6, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16376, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130786 payload_rows_remote=114500 dup_rows=286 dup_rows_remote=286 dedup_ratio=0.002182 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 10, 0, 1, 0, 0, 0, 0, 0, 69, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8103, 0, 0, 0, 8189, 0, 0, 10, 8138, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3, 0] payload_rows_per_dst=[16384, 16372, 16384, 16374, 16382, 16228, 16286, 16376] dup_rows_per_dst=[14, 6, 264, 0, 0, 0, 0, 2] combine_rows_per_src=[16384, 16372, 16384, 16374, 16382, 16228, 16286, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129102 payload_rows_remote=112726 dup_rows=1970 dup_rows_remote=1732 dedup_ratio=0.015030 router_tokens_per_expert=[5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 69, 0, 90, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 369, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 7942, 0, 0, 43, 0, 13, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 48, 8141, 0, 2, 0, 0, 0, 19, 2, 0, 0, 0, 0, 0, 0, 2, 0, 8156, 2, 0, 0, 8095, 0, 0, 0, 8107, 0, 0, 3, 0, 26, 0, 47, 0, 0, 0, 0, 0, 8, 0, 0, 7819, 0, 27, 0, 0, 0, 0, 0, 21, 0, 2, 30, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16300, 16356, 16290, 16098, 15580, 16368, 15734] dup_rows_per_dst=[238, 610, 824, 92, 128, 0, 14, 64] combine_rows_per_src=[16376, 16300, 16356, 16290, 16098, 15580, 16368, 15734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=919519232 dispatch_meta_bytes=456352 combine_payload_bytes=919519232 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919975584 ingress_bytes=995016704 link_bytes=995016704 token_bytes=1839494816 weight_movement_bytes=75497472 total_nvlink_bytes=1914992288 weight_to_token_ratio=0.041043 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.261821240e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=938672128 dispatch_meta_bytes=458744 combine_payload_bytes=938672128 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939130872 ingress_bytes=1014169600 link_bytes=1014169600 token_bytes=1877803000 weight_movement_bytes=75497472 total_nvlink_bytes=1953300472 weight_to_token_ratio=0.040205 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286109808e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458728 combine_payload_bytes=939425792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939884520 ingress_bytes=1014923264 link_bytes=1014923264 token_bytes=1879310312 weight_movement_bytes=75497472 total_nvlink_bytes=1954807784 weight_to_token_ratio=0.040173 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287065560e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130866 payload_rows_remote=114482 dup_rows=206 dup_rows_remote=206 dedup_ratio=0.001572 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 4, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 55, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8163, 0, 0, 0, 8191, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8186, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16332, 16384, 16360, 16384, 16328, 16318, 16376] dup_rows_per_dst=[58, 0, 148, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16332, 16384, 16360, 16384, 16328, 16318, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128394 payload_rows_remote=112010 dup_rows=2678 dup_rows_remote=2670 dedup_ratio=0.020432 router_tokens_per_expert=[0, 0, 0, 7557, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7808, 0, 223, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 7287, 0, 2, 7, 0, 0, 0, 0, 0, 0, 0, 406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[15188, 16384, 16384, 16050, 16384, 16384, 15236, 16384] dup_rows_per_dst=[0, 2358, 126, 12, 0, 8, 174, 0] combine_rows_per_src=[15188, 16384, 16384, 16050, 16384, 16384, 15236, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128930 payload_rows_remote=112570 dup_rows=2142 dup_rows_remote=2142 dedup_ratio=0.016342 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 337, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 4, 0, 0, 8188, 0, 8151, 0, 0, 0, 8131, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7519, 1076, 0, 0, 0, 1, 0, 0, 2, 7047, 0, 5, 0, 0, 0, 0, 0, 0, 8192, 0, 348, 0, 0, 94, 59, 0, 0, 0, 2, 0] payload_rows_per_dst=[14760, 16264, 16360, 16306, 16376, 16384, 16096, 16384] dup_rows_per_dst=[8, 0, 0, 8, 0, 0, 1120, 1006] combine_rows_per_src=[14760, 16264, 16360, 16306, 16376, 16384, 16096, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120784 payload_rows_remote=104474 dup_rows=10288 dup_rows_remote=10272 dedup_ratio=0.078491 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8186, 0, 0, 0, 15, 8139, 0, 8192, 0, 8, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2840, 259, 0, 0, 0, 2, 0, 0, 0, 8186, 0, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5059, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16372, 16376, 16380, 16310, 16384, 16380, 6198, 16384] dup_rows_per_dst=[0, 138, 6, 16, 0, 0, 10, 10118] combine_rows_per_src=[16372, 16376, 16380, 16310, 16384, 16380, 6198, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123734 payload_rows_remote=107352 dup_rows=7338 dup_rows_remote=1640 dedup_ratio=0.055984 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7748, 0, 8190, 0, 1012, 0, 0, 0, 8160, 0, 0, 0, 215, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1, 7355, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 81, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16334, 16382, 10782, 16384, 16374, 14712, 16382] dup_rows_per_dst=[0, 4, 0, 772, 864, 0, 0, 5698] combine_rows_per_src=[16384, 16334, 16382, 10782, 16384, 16374, 14712, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127012 payload_rows_remote=110628 dup_rows=4060 dup_rows_remote=4060 dedup_ratio=0.030975 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8158, 0, 8192, 0, 67, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 12, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16266, 16384, 16384, 16368, 16384, 16384, 16384, 12458] dup_rows_per_dst=[0, 0, 0, 82, 0, 6, 3972, 0] combine_rows_per_src=[16266, 16384, 16384, 16368, 16384, 16384, 16384, 12458] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=937984000 dispatch_meta_bytes=459144 combine_payload_bytes=937984000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938443144 ingress_bytes=1013481472 link_bytes=1013481472 token_bytes=1876427144 weight_movement_bytes=75497472 total_nvlink_bytes=1951924616 weight_to_token_ratio=0.040235 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285237165e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=923451392 dispatch_meta_bytes=457832 combine_payload_bytes=923451392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=923909224 ingress_bytes=998948864 link_bytes=998948864 token_bytes=1847360616 weight_movement_bytes=75497472 total_nvlink_bytes=1922858088 weight_to_token_ratio=0.040868 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.266807772e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=917585920 dispatch_meta_bytes=458720 combine_payload_bytes=917585920 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=918044640 ingress_bytes=1068580864 link_bytes=1068580864 token_bytes=1835630560 weight_movement_bytes=150994944 total_nvlink_bytes=1986625504 weight_to_token_ratio=0.082258 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=62208 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.355110950e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=937836544 dispatch_meta_bytes=458752 combine_payload_bytes=937836544 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=938295296 ingress_bytes=1164328960 link_bytes=1164328960 token_bytes=1876131840 weight_movement_bytes=226492416 total_nvlink_bytes=2102624256 weight_to_token_ratio=0.120723 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.476533013e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=922173440 dispatch_meta_bytes=458848 combine_payload_bytes=922173440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=922632288 ingress_bytes=997670912 link_bytes=997670912 token_bytes=1844805728 weight_movement_bytes=75497472 total_nvlink_bytes=1920303200 weight_to_token_ratio=0.040924 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265187149e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=879427584 dispatch_meta_bytes=435968 combine_payload_bytes=879427584 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=879863552 ingress_bytes=1030422528 link_bytes=1030422528 token_bytes=1759291136 weight_movement_bytes=150994944 total_nvlink_bytes=1910286080 weight_to_token_ratio=0.085827 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=55424 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.306720808e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=855851008 dispatch_meta_bytes=458984 combine_payload_bytes=855851008 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=856309992 ingress_bytes=1308835840 link_bytes=1308835840 token_bytes=1712161000 weight_movement_bytes=452984832 total_nvlink_bytes=2165145832 weight_to_token_ratio=0.264569 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.049632 gemm_active_groups=10 gemm_peak_group_rows=48512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.659788078e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=906264576 dispatch_meta_bytes=458752 combine_payload_bytes=906264576 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=906723328 ingress_bytes=1057259520 link_bytes=1057259520 token_bytes=1812987904 weight_movement_bytes=150994944 total_nvlink_bytes=1963982848 weight_to_token_ratio=0.083285 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.340753892e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20880160360 prefetch_bytes=981467136 reduce_bytes=1283457024 weight_movement_bytes=2264924160 egress_bytes=10442815080 ingress_bytes=12702269440 link_bytes=12702269440 total_nvlink_bytes=23145084520 weight_to_token_ratio=0.108473 prefetch_share_of_ingress=0.077267 dedup_ratio=0.068326 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=17 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044884 gemm_inner_pad_upper_bound=16510 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.342355469e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20004023784 prefetch_bytes=830472192 reduce_bytes=377487360 weight_movement_bytes=1207959552 egress_bytes=10004704744 ingress_bytes=11207278592 link_bytes=11207278592 total_nvlink_bytes=21211983336 weight_to_token_ratio=0.060386 prefetch_share_of_ingress=0.074101 dedup_ratio=0.099111 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=5 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043811 gemm_inner_pad_upper_bound=17780 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.184367233e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21426462824 prefetch_bytes=905969664 reduce_bytes=1283457024 weight_movement_bytes=2189426688 egress_bytes=10715979880 ingress_bytes=12899909632 link_bytes=12899909632 total_nvlink_bytes=23615889512 weight_to_token_ratio=0.102183 prefetch_share_of_ingress=0.070231 dedup_ratio=0.046303 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=17 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=17526 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.363241768e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20630356968 prefetch_bytes=905969664 reduce_bytes=905969664 weight_movement_bytes=1811939328 egress_bytes=10317874152 ingress_bytes=12124422144 link_bytes=12124422144 total_nvlink_bytes=22442296296 weight_to_token_ratio=0.087829 prefetch_share_of_ingress=0.074723 dedup_ratio=0.076415 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=12 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045190 gemm_inner_pad_upper_bound=16256 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.281289493e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130854 payload_rows_remote=114590 dup_rows=218 dup_rows_remote=160 dedup_ratio=0.001663 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 5611, 0, 8188, 0, 2108, 0, 0, 0, 8187, 1, 0, 0, 500, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 4, 8190, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8104, 57, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16372, 16374, 16374, 16320, 16382, 16384, 16264, 16384] dup_rows_per_dst=[6, 0, 0, 120, 2, 28, 58, 4] combine_rows_per_src=[16372, 16374, 16374, 16320, 16382, 16384, 16264, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113184 payload_rows_remote=96882 dup_rows=17888 dup_rows_remote=11650 dedup_ratio=0.136475 router_tokens_per_expert=[0, 5, 0, 0, 12, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 4, 0, 0, 0, 24, 0, 0, 7841, 0, 27, 19, 0, 8179, 0, 7767, 0, 0, 0, 0, 0, 7843, 0, 0, 107, 2159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 198, 0, 8136, 0, 312, 0, 0, 0, 0, 0, 0, 0, 22, 0, 20, 30, 0, 0, 7929, 0, 0, 0, 0, 0, 0, 0, 0, 6431, 13, 0, 0, 0, 26, 1, 4, 0, 8027, 0, 0, 0, 17, 268, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16302, 16310, 16104, 16372, 15538, 15878, 16080, 600] dup_rows_per_dst=[6238, 106, 336, 4456, 0, 428, 6292, 32] combine_rows_per_src=[16302, 16310, 16104, 16372, 15538, 15878, 16080, 600] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[2, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126700 payload_rows_remote=110326 dup_rows=4372 dup_rows_remote=4372 dedup_ratio=0.033356 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8188, 0, 0, 0, 0, 8192, 0, 8187, 0, 9, 0, 0, 0, 8191, 0, 0, 36, 1008, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 8188, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7182, 0, 0, 2, 0, 0, 0, 0, 0, 7055, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 1104, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14124, 16384, 16378, 16384, 16374, 16384, 14368, 16304] dup_rows_per_dst=[0, 6, 0, 2106, 0, 52, 0, 2208] combine_rows_per_src=[14124, 16384, 16378, 16384, 16374, 16384, 14368, 16304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131020 payload_rows_remote=114636 dup_rows=52 dup_rows_remote=52 dedup_ratio=0.000397 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16336, 16384, 16384, 16384] dup_rows_per_dst=[4, 0, 48, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16336, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938721280 dispatch_meta_bytes=459000 combine_payload_bytes=938721280 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939180280 ingress_bytes=1165213696 link_bytes=1165213696 token_bytes=1877901560 weight_movement_bytes=226492416 total_nvlink_bytes=2104393976 weight_to_token_ratio=0.120609 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=52224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.477654983e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=793657344 dispatch_meta_bytes=434128 combine_payload_bytes=793657344 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=794091472 ingress_bytes=944652288 link_bytes=944652288 token_bytes=1587748816 weight_movement_bytes=150994944 total_nvlink_bytes=1738743760 weight_to_token_ratio=0.095100 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=60416 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.197952071e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=903790592 dispatch_meta_bytes=458792 combine_payload_bytes=903790592 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=904249384 ingress_bytes=1054785536 link_bytes=1054785536 token_bytes=1808039976 weight_movement_bytes=150994944 total_nvlink_bytes=1959034920 weight_to_token_ratio=0.083513 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.337616532e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939098112 dispatch_meta_bytes=458752 combine_payload_bytes=939098112 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=939556864 ingress_bytes=1316585472 link_bytes=1316585472 token_bytes=1878654976 weight_movement_bytes=377487360 total_nvlink_bytes=2256142336 weight_to_token_ratio=0.200935 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=64512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.669615702e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21232010640 prefetch_bytes=830472192 reduce_bytes=452984832 weight_movement_bytes=1283457024 egress_bytes=10618733968 ingress_bytes=11896733696 link_bytes=11896733696 total_nvlink_bytes=22515467664 weight_to_token_ratio=0.060449 prefetch_share_of_ingress=0.069807 dedup_ratio=0.053238 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=6 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044424 gemm_inner_pad_upper_bound=18034 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.257227743e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19124880320 prefetch_bytes=905969664 reduce_bytes=528482304 weight_movement_bytes=1434451968 egress_bytes=9565078464 ingress_bytes=10994253824 link_bytes=10994253824 total_nvlink_bytes=20559332288 weight_to_token_ratio=0.075004 prefetch_share_of_ingress=0.082404 dedup_ratio=0.134026 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=7 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045496 gemm_inner_pad_upper_bound=16764 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.161855117e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21551168008 prefetch_bytes=905969664 reduce_bytes=1132462080 weight_movement_bytes=2038431744 egress_bytes=10778327560 ingress_bytes=12811272192 link_bytes=12811272192 total_nvlink_bytes=23589599752 weight_to_token_ratio=0.094586 prefetch_share_of_ingress=0.070717 dedup_ratio=0.041454 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=15 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045343 gemm_inner_pad_upper_bound=16002 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.353874705e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126914 payload_rows_remote=110530 dup_rows=4158 dup_rows_remote=4152 dedup_ratio=0.031723 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 5976, 0, 8192, 0, 1284, 0, 0, 0, 8192, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7796, 1963, 0, 0, 0, 0, 0, 0, 0, 7120, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 263, 0, 0, 100, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14240, 16384, 16368, 14522, 16384, 16384, 16248, 16384] dup_rows_per_dst=[0, 8, 0, 148, 0, 6, 3270, 726] combine_rows_per_src=[14240, 16384, 16368, 14522, 16384, 16384, 16248, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=905461760 dispatch_meta_bytes=458728 combine_payload_bytes=905461760 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=905920488 ingress_bytes=1056456704 link_bytes=1056456704 token_bytes=1811382248 weight_movement_bytes=150994944 total_nvlink_bytes=1962377192 weight_to_token_ratio=0.083359 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.339735808e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21045134752 prefetch_bytes=830472192 reduce_bytes=1132462080 weight_movement_bytes=1962934272 egress_bytes=10525296032 ingress_bytes=12482772992 link_bytes=12482772992 total_nvlink_bytes=23008069024 weight_to_token_ratio=0.093273 prefetch_share_of_ingress=0.066529 dedup_ratio=0.060359 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=15 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=17907 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.319159436e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none + [2026-08-29 23:35:10] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 371.7 | throughput per GPU (TFLOP/s/GPU): 486.5 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.338105E+01 | loss scale: 1.0 | grad norm: 20.297 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104902 payload_rows_remote=89054 dup_rows=26170 dup_rows_remote=21254 dedup_ratio=0.199661 router_tokens_per_expert=[20, 0, 0, 6, 113, 165, 1481, 0, 7, 3017, 1, 0, 21, 1, 25, 2, 1, 216, 0, 6, 758, 0, 0, 1850, 4689, 1, 0, 8127, 1361, 5715, 1, 109, 16, 25, 106, 0, 4, 443, 0, 6, 71, 2, 927, 490, 0, 448, 39, 7805, 0, 1, 26, 1, 0, 4, 0, 3, 42, 1, 18, 169, 0, 2, 0, 8, 1964, 0, 0, 0, 406, 0, 40, 2, 721, 0, 686, 1095, 1, 5, 137, 0, 2, 1, 0, 3, 28, 0, 0, 356, 0, 189, 0, 4, 12, 1, 0, 3, 0, 196, 2, 0, 6, 8, 8, 1, 22, 674, 268, 139, 0, 65, 5675, 0, 0, 5559, 2, 0, 1, 25, 2519, 0, 2, 1, 5749, 602, 0, 1, 1, 4] payload_rows_per_dst=[14698, 12982, 15848, 16282, 14466, 11880, 12590, 6156] dup_rows_per_dst=[6518, 5002, 4916, 522, 6766, 748, 1538, 160] combine_rows_per_src=[14698, 12982, 15848, 16282, 14466, 11880, 12590, 6156] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=92332 payload_rows_remote=77486 dup_rows=38740 dup_rows_remote=28560 dedup_ratio=0.295563 router_tokens_per_expert=[8, 0, 0, 10, 91, 250, 1215, 0, 0, 1928, 1, 3, 5, 0, 55, 5, 0, 192, 0, 15, 556, 0, 5, 1493, 4718, 0, 2, 8151, 1665, 6093, 10, 620, 9, 24, 42, 0, 0, 169, 0, 2, 14, 2, 981, 885, 0, 451, 2, 7637, 2, 1, 39, 0, 4, 0, 0, 0, 34, 3, 33, 149, 0, 1, 0, 5, 2527, 0, 0, 0, 192, 1, 16, 1, 626, 1, 983, 1374, 11, 5, 51, 0, 2, 1, 0, 0, 75, 0, 0, 609, 5, 101, 0, 4, 2, 0, 0, 6, 0, 134, 7, 3, 6, 2, 0, 0, 83, 733, 357, 211, 1, 58, 5304, 2, 3, 5498, 0, 1, 0, 8, 2061, 0, 3, 3, 6339, 539, 0, 1, 0, 6] payload_rows_per_dst=[6564, 7272, 7240, 16332, 9458, 14824, 14846, 15796] dup_rows_per_dst=[578, 1844, 1972, 512, 2118, 8408, 10180, 13128] combine_rows_per_src=[6564, 7272, 7240, 16332, 9458, 14824, 14846, 15796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=106454 payload_rows_remote=97560 dup_rows=24618 dup_rows_remote=22838 dedup_ratio=0.187820 router_tokens_per_expert=[2, 0, 0, 7, 150, 143, 1087, 4, 6, 1447, 1, 2, 4, 0, 24, 6, 1, 173, 0, 13, 921, 0, 0, 1732, 4501, 3, 0, 8136, 1470, 5949, 2, 520, 9, 20, 153, 0, 1, 219, 1, 10, 20, 2, 506, 436, 2, 1208, 3, 7712, 0, 1, 12, 3, 0, 7, 0, 6, 142, 1, 11, 158, 0, 2, 0, 20, 1791, 0, 0, 0, 350, 0, 4, 7, 793, 0, 668, 1492, 23, 14, 195, 0, 0, 0, 0, 0, 62, 0, 0, 1015, 0, 145, 1, 18, 0, 4, 0, 6, 1, 528, 0, 4, 9, 1, 0, 3, 90, 741, 270, 127, 4, 128, 5498, 0, 0, 6019, 3, 0, 0, 5, 2098, 0, 2, 2, 6037, 401, 1, 0, 5, 2] payload_rows_per_dst=[13796, 12430, 15724, 16302, 8894, 12906, 12678, 13724] dup_rows_per_dst=[4044, 5316, 4880, 696, 1780, 2420, 2130, 3352] combine_rows_per_src=[13796, 12430, 15724, 16302, 8894, 12906, 12678, 13724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105382 payload_rows_remote=89026 dup_rows=25690 dup_rows_remote=25152 dedup_ratio=0.195999 router_tokens_per_expert=[13, 1, 0, 6, 65, 136, 1447, 0, 0, 1682, 0, 0, 2, 3, 32, 3, 0, 281, 0, 7, 761, 0, 2, 1769, 4200, 1, 0, 8165, 1667, 5661, 1, 712, 3, 16, 130, 0, 0, 85, 0, 4, 24, 0, 647, 604, 0, 493, 2, 7540, 0, 0, 16, 0, 0, 1, 0, 0, 105, 0, 23, 114, 1, 0, 0, 22, 1653, 0, 0, 0, 337, 0, 10, 1, 429, 0, 1158, 977, 8, 9, 87, 0, 0, 0, 2, 2, 66, 0, 0, 581, 0, 196, 0, 0, 7, 0, 2, 0, 0, 267, 1, 0, 4, 1, 2, 1, 147, 515, 444, 150, 0, 107, 6122, 0, 1, 6458, 0, 2, 0, 6, 2276, 0, 0, 0, 6544, 516, 0, 0, 0, 0] payload_rows_per_dst=[14812, 12874, 15402, 16356, 14522, 11912, 13144, 6360] dup_rows_per_dst=[5056, 5928, 3694, 538, 6560, 1122, 2378, 414] combine_rows_per_src=[14812, 12874, 15402, 16356, 14522, 11912, 13144, 6360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102494 payload_rows_remote=89884 dup_rows=28578 dup_rows_remote=23720 dedup_ratio=0.218033 router_tokens_per_expert=[5, 0, 0, 11, 115, 67, 1336, 0, 0, 2305, 5, 2, 21, 4, 54, 18, 0, 386, 0, 19, 663, 0, 3, 1356, 4475, 2, 1, 8099, 1373, 5398, 0, 456, 24, 13, 97, 0, 4, 292, 0, 3, 22, 1, 1304, 481, 1, 498, 37, 7724, 0, 1, 32, 0, 0, 0, 0, 1, 68, 1, 11, 250, 0, 1, 0, 12, 2291, 0, 0, 0, 427, 0, 3, 0, 566, 0, 1040, 1474, 15, 3, 73, 3, 18, 8, 2, 0, 42, 0, 1, 551, 6, 226, 0, 3, 28, 4, 7, 0, 2, 262, 4, 1, 16, 5, 0, 0, 143, 601, 474, 330, 1, 135, 5590, 0, 1, 5878, 0, 0, 0, 16, 1565, 1, 0, 10, 6255, 378, 2, 17, 1, 5] payload_rows_per_dst=[14658, 12610, 15718, 16216, 15064, 11488, 12778, 3962] dup_rows_per_dst=[5738, 4858, 5284, 736, 8482, 1100, 2350, 30] combine_rows_per_src=[14658, 12610, 15718, 16216, 15064, 11488, 12778, 3962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=88728 payload_rows_remote=72552 dup_rows=42344 dup_rows_remote=17158 dedup_ratio=0.323059 router_tokens_per_expert=[6, 0, 0, 21, 63, 162, 1296, 0, 0, 2564, 6, 0, 1, 1, 37, 17, 1, 283, 0, 0, 670, 0, 4, 1579, 4738, 0, 0, 8188, 1924, 6047, 0, 603, 2, 2, 165, 0, 4, 263, 0, 10, 21, 0, 604, 1125, 0, 444, 0, 7792, 0, 2, 21, 0, 0, 2, 0, 1, 161, 0, 16, 51, 0, 0, 0, 7, 1875, 0, 0, 0, 383, 0, 1, 3, 552, 0, 577, 952, 2, 3, 81, 0, 0, 1, 0, 3, 28, 0, 0, 607, 0, 147, 0, 3, 4, 0, 7, 0, 0, 147, 0, 0, 7, 4, 0, 0, 95, 344, 467, 111, 1, 51, 5541, 1, 0, 6209, 0, 0, 0, 9, 1438, 0, 0, 0, 6556, 421, 0, 1, 0, 0] payload_rows_per_dst=[7602, 8118, 4750, 16378, 7824, 14972, 12908, 16176] dup_rows_per_dst=[746, 2010, 530, 520, 1034, 8198, 4120, 25186] combine_rows_per_src=[7602, 8118, 4750, 16378, 7824, 14972, 12908, 16176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=729530368 dispatch_meta_bytes=441232 combine_payload_bytes=729530368 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=729971600 ingress_bytes=956022784 link_bytes=956022784 token_bytes=1459501968 weight_movement_bytes=226492416 total_nvlink_bytes=1685994384 weight_to_token_ratio=0.155185 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.038603 gemm_active_groups=17 gemm_peak_group_rows=46336 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212371460e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102444 payload_rows_remote=88672 dup_rows=28628 dup_rows_remote=22366 dedup_ratio=0.218414 router_tokens_per_expert=[14, 0, 0, 25, 95, 180, 1445, 0, 0, 2452, 1, 4, 10, 2, 49, 1, 0, 226, 0, 23, 616, 2, 0, 2345, 3969, 0, 0, 8161, 1898, 5146, 4, 605, 1, 28, 138, 2, 1, 253, 0, 7, 19, 0, 387, 501, 0, 651, 4, 7855, 0, 1, 12, 0, 0, 3, 0, 0, 60, 1, 18, 182, 0, 0, 0, 14, 2709, 4, 0, 0, 503, 0, 1, 0, 603, 0, 663, 1286, 20, 0, 40, 0, 0, 0, 0, 1, 20, 0, 0, 683, 0, 176, 1, 2, 10, 1, 0, 8, 0, 68, 0, 0, 11, 3, 5, 0, 119, 603, 217, 103, 4, 64, 6048, 0, 1, 5632, 0, 2, 0, 8, 1877, 0, 1, 7, 5946, 672, 0, 2, 0, 1] payload_rows_per_dst=[11032, 8550, 15858, 16334, 9490, 13772, 12878, 14530] dup_rows_per_dst=[3252, 2888, 3836, 570, 2168, 6262, 1612, 8040] combine_rows_per_src=[11032, 8550, 15858, 16334, 9490, 13772, 12878, 14530] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=634765312 dispatch_meta_bytes=424184 combine_payload_bytes=634765312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=635189496 ingress_bytes=710262784 link_bytes=710262784 token_bytes=1269954808 weight_movement_bytes=75497472 total_nvlink_bytes=1345452280 weight_to_token_ratio=0.059449 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.034926 gemm_active_groups=18 gemm_peak_group_rows=46208 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.007131872e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=799211520 dispatch_meta_bytes=481592 combine_payload_bytes=799211520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=799693112 ingress_bytes=874708992 link_bytes=874708992 token_bytes=1598904632 weight_movement_bytes=75497472 total_nvlink_bytes=1674402104 weight_to_token_ratio=0.047218 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.042279 gemm_active_groups=16 gemm_peak_group_rows=23808 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.109254126e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=729300992 dispatch_meta_bytes=456712 combine_payload_bytes=729300992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=729757704 ingress_bytes=804798464 link_bytes=804798464 token_bytes=1459058696 weight_movement_bytes=75497472 total_nvlink_bytes=1534556168 weight_to_token_ratio=0.051744 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.034926 gemm_active_groups=17 gemm_peak_group_rows=63232 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.020597736e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=736329728 dispatch_meta_bytes=454416 combine_payload_bytes=736329728 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=736784144 ingress_bytes=1038319616 link_bytes=1038319616 token_bytes=1473113872 weight_movement_bytes=301989888 total_nvlink_bytes=1775103760 weight_to_token_ratio=0.205001 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.040441 gemm_active_groups=16 gemm_peak_group_rows=21120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.316735428e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=594345984 dispatch_meta_bytes=358840 combine_payload_bytes=594345984 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=594704824 ingress_bytes=820838400 link_bytes=820838400 token_bytes=1189050808 weight_movement_bytes=226492416 total_nvlink_bytes=1415543224 weight_to_token_ratio=0.190482 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.034926 gemm_active_groups=18 gemm_peak_group_rows=24064 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.040938633e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=726401024 dispatch_meta_bytes=444152 combine_payload_bytes=726401024 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=726845176 ingress_bytes=877395968 link_bytes=877395968 token_bytes=1453246200 weight_movement_bytes=150994944 total_nvlink_bytes=1604241144 weight_to_token_ratio=0.103902 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.034926 gemm_active_groups=18 gemm_peak_group_rows=45184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.112661590e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98330 payload_rows_remote=82742 dup_rows=32742 dup_rows_remote=26352 dedup_ratio=0.249802 router_tokens_per_expert=[3, 0, 0, 9, 115, 142, 1528, 0, 0, 2039, 1, 0, 3, 2, 27, 14, 0, 105, 0, 2, 1089, 0, 9, 2080, 3668, 0, 0, 8113, 2010, 5072, 3, 947, 7, 19, 228, 0, 7, 199, 0, 1, 96, 0, 587, 419, 0, 512, 6, 7968, 0, 5, 12, 0, 0, 2, 0, 0, 93, 0, 19, 82, 0, 0, 4, 2, 2168, 0, 0, 0, 445, 0, 3, 2, 724, 0, 642, 665, 14, 1, 73, 0, 0, 1, 0, 0, 28, 0, 1, 341, 0, 69, 0, 14, 2, 1, 1, 0, 0, 203, 1, 0, 9, 1, 1, 0, 80, 186, 320, 117, 0, 124, 6429, 0, 1, 6486, 0, 0, 0, 17, 1572, 0, 0, 2, 7106, 432, 0, 4, 0, 1] payload_rows_per_dst=[15588, 14348, 12778, 12572, 15134, 10506, 13424, 3980] dup_rows_per_dst=[6390, 14042, 2642, 206, 7312, 554, 1518, 78] combine_rows_per_src=[15588, 14348, 12778, 12572, 15134, 10506, 13424, 3980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123820 payload_rows_remote=107438 dup_rows=7252 dup_rows_remote=7248 dedup_ratio=0.055328 router_tokens_per_expert=[8165, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 4, 96, 328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 8, 0, 1, 0, 8187, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 4827, 0, 22, 0, 0, 0, 5586, 0, 0, 0, 0, 0, 5329, 0, 0, 0, 0] payload_rows_per_dst=[16334, 16384, 16382, 16384, 16374, 16366, 9660, 15936] dup_rows_per_dst=[2, 2, 4, 850, 0, 454, 2, 5938] combine_rows_per_src=[16334, 16384, 16382, 16384, 16374, 16366, 9660, 15936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116742 payload_rows_remote=101662 dup_rows=14330 dup_rows_remote=3556 dedup_ratio=0.109329 router_tokens_per_expert=[8129, 0, 15, 0, 1, 0, 0, 1, 3, 5, 0, 10, 0, 0, 0, 0, 4, 0, 1, 0, 8181, 14, 0, 8171, 0, 0, 2, 3, 0, 0, 0, 2, 1, 0, 1, 0, 0, 8, 0, 0, 8, 0, 0, 0, 1, 0, 7, 1, 4, 18, 449, 0, 0, 0, 0, 0, 7, 2, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 1, 4, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 8175, 0, 1, 31, 0, 0, 1, 8179, 0, 2895, 0, 0, 4, 0, 1, 0, 68, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 3, 6132, 2, 0, 3, 0, 0, 4150, 0, 1, 3, 0, 0, 2664, 0, 0, 0, 4] payload_rows_per_dst=[16302, 16366, 16380, 16380, 16374, 7374, 15080, 12486] dup_rows_per_dst=[26, 28, 36, 890, 4, 1404, 10774, 1168] combine_rows_per_src=[16302, 16366, 16380, 16380, 16374, 7374, 15080, 12486] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126594 payload_rows_remote=110210 dup_rows=4478 dup_rows_remote=4476 dedup_ratio=0.034164 router_tokens_per_expert=[8182, 0, 1, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 8184, 17, 8, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 7, 49, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8153, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 2, 2, 0, 0, 4, 8192, 0, 1029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 6331, 0, 1, 0, 0, 0, 6964, 0, 0, 0, 1, 0, 1995, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16382, 16384, 16330, 16384, 16382, 12680, 15674] dup_rows_per_dst=[4, 36, 6, 110, 2, 2070, 2, 2248] combine_rows_per_src=[16378, 16382, 16384, 16330, 16384, 16382, 12680, 15674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127794 payload_rows_remote=111410 dup_rows=3278 dup_rows_remote=2402 dedup_ratio=0.025009 router_tokens_per_expert=[8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 9, 0, 0, 0, 8192, 0, 0, 8186, 43, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 422, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 621, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7591, 0, 1, 0, 0, 0, 4382, 0, 0, 0, 0, 0, 3302, 0, 0, 0, 0] payload_rows_per_dst=[16338, 16378, 16384, 16384, 16384, 16384, 15182, 14360] dup_rows_per_dst=[2, 100, 30, 876, 0, 1260, 0, 1010] combine_rows_per_src=[16338, 16378, 16384, 16384, 16384, 16384, 15182, 14360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122854 payload_rows_remote=106470 dup_rows=8218 dup_rows_remote=8210 dedup_ratio=0.062698 router_tokens_per_expert=[8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 168, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 28, 0, 2, 0, 8192, 0, 606, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4283, 0, 0, 0, 0, 0, 6418, 0, 1, 0, 0, 0, 4614, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16384, 16384, 16380, 16384, 16384, 8566, 16002] dup_rows_per_dst=[4, 8, 0, 880, 0, 1262, 0, 6064] combine_rows_per_src=[16370, 16384, 16384, 16380, 16384, 16384, 8566, 16002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=677822464 dispatch_meta_bytes=436376 combine_payload_bytes=677822464 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=678258840 ingress_bytes=753319936 link_bytes=753319936 token_bytes=1356081304 weight_movement_bytes=75497472 total_nvlink_bytes=1431578776 weight_to_token_ratio=0.055673 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=34560 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.553157166e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=880132096 dispatch_meta_bytes=458744 combine_payload_bytes=880132096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=880590840 ingress_bytes=955629568 link_bytes=955629568 token_bytes=1760722936 weight_movement_bytes=75497472 total_nvlink_bytes=1836220408 weight_to_token_ratio=0.042879 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.034926 gemm_active_groups=15 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.211872807e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=832815104 dispatch_meta_bytes=420872 combine_payload_bytes=832815104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=833235976 ingress_bytes=908312576 link_bytes=908312576 token_bytes=1666051080 weight_movement_bytes=75497472 total_nvlink_bytes=1741548552 weight_to_token_ratio=0.045315 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=50560 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.151868201e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109754 payload_rows_remote=100638 dup_rows=21318 dup_rows_remote=21160 dedup_ratio=0.162643 router_tokens_per_expert=[8179, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8190, 1, 0, 8189, 24, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5, 2, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 8189, 0, 4136, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7593, 0, 2, 0, 1, 0, 2605, 1, 1, 0, 0, 0, 2016, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16360, 16382, 7082, 16382, 11684, 16384, 9116] dup_rows_per_dst=[2, 50, 16, 22, 2, 5872, 15196, 158] combine_rows_per_src=[16364, 16360, 16382, 7082, 16382, 11684, 16384, 9116] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=902840320 dispatch_meta_bytes=458744 combine_payload_bytes=902840320 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=903299064 ingress_bytes=1053835264 link_bytes=1053835264 token_bytes=1806139384 weight_movement_bytes=150994944 total_nvlink_bytes=1957134328 weight_to_token_ratio=0.083601 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.336411453e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127662 payload_rows_remote=111292 dup_rows=3410 dup_rows_remote=2398 dedup_ratio=0.026016 router_tokens_per_expert=[8091, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 1, 0, 0, 3, 0, 75, 0, 0, 0, 8191, 0, 0, 8182, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 40, 0, 6, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 36, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 8191, 0, 504, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 93, 0, 0, 6960, 0, 4, 0, 0, 0, 5284, 0, 2, 0, 0, 0, 3468, 0, 0, 0, 0] payload_rows_per_dst=[16188, 16378, 16384, 16382, 16382, 16370, 14052, 15526] dup_rows_per_dst=[8, 164, 104, 78, 0, 1012, 54, 1990] combine_rows_per_src=[16188, 16378, 16384, 16382, 16382, 16370, 14052, 15526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=912670720 dispatch_meta_bytes=455248 combine_payload_bytes=912670720 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=913125968 ingress_bytes=1063665664 link_bytes=1063665664 token_bytes=1825796688 weight_movement_bytes=150994944 total_nvlink_bytes=1976791632 weight_to_token_ratio=0.082701 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=60800 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.348877784e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=872202240 dispatch_meta_bytes=458720 combine_payload_bytes=872202240 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=872660960 ingress_bytes=1098694656 link_bytes=1098694656 token_bytes=1744863200 weight_movement_bytes=226492416 total_nvlink_bytes=1971355616 weight_to_token_ratio=0.129805 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.393299477e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=824426496 dispatch_meta_bytes=487192 combine_payload_bytes=824426496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=824913688 ingress_bytes=975421440 link_bytes=975421440 token_bytes=1649340184 weight_movement_bytes=150994944 total_nvlink_bytes=1800335128 weight_to_token_ratio=0.091549 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=36992 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.236971687e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=911704064 dispatch_meta_bytes=454760 combine_payload_bytes=911704064 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=912158824 ingress_bytes=1213693952 link_bytes=1213693952 token_bytes=1823862888 weight_movement_bytes=301989888 total_nvlink_bytes=2125852776 weight_to_token_ratio=0.165577 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.042279 gemm_active_groups=13 gemm_peak_group_rows=50560 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.539134771e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130806 payload_rows_remote=114504 dup_rows=266 dup_rows_remote=266 dedup_ratio=0.002029 router_tokens_per_expert=[8192, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 8192, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16378, 16302, 16382, 16384, 16208, 16384, 16384] dup_rows_per_dst=[16, 134, 0, 18, 4, 0, 58, 36] combine_rows_per_src=[16384, 16378, 16302, 16382, 16384, 16208, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130488 payload_rows_remote=114118 dup_rows=584 dup_rows_remote=192 dedup_ratio=0.004456 router_tokens_per_expert=[8036, 3, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 8141, 8172, 0, 0, 1, 0, 0, 5, 8167, 0, 0, 0, 2, 0, 0, 0, 0, 0, 47, 10, 0, 0, 0, 12, 0, 0, 0, 0, 1, 4, 0, 8171, 0, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8167, 9, 1, 8101, 0, 0, 0, 0, 10, 0, 0, 0, 0, 5, 0, 0, 0, 0, 191, 0, 11, 0, 6, 0, 1, 0, 0, 0, 0, 20, 0, 3, 9, 0, 0, 1, 0, 0, 0, 0, 8181, 0, 6] payload_rows_per_dst=[16110, 16302, 16362, 16368, 16360, 16232, 16370, 16384] dup_rows_per_dst=[4, 8, 82, 28, 0, 14, 392, 56] combine_rows_per_src=[16110, 16302, 16362, 16368, 16360, 16232, 16370, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123734 payload_rows_remote=107430 dup_rows=7338 dup_rows_remote=6892 dedup_ratio=0.055984 router_tokens_per_expert=[8017, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 2, 0, 59, 0, 0, 0, 8191, 0, 0, 8181, 2, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 6, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 621, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 19, 0, 0, 0, 8188, 0, 178, 21, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6719, 0, 12, 0, 0, 0, 3751, 2, 2, 0, 0, 0, 5144, 1, 0, 0, 0] payload_rows_per_dst=[16304, 15948, 16196, 16370, 16384, 16382, 13438, 12712] dup_rows_per_dst=[446, 104, 0, 5674, 16, 412, 2, 684] combine_rows_per_src=[16304, 15948, 16196, 16370, 16384, 16382, 13438, 12712] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130670 payload_rows_remote=114286 dup_rows=402 dup_rows_remote=388 dedup_ratio=0.003067 router_tokens_per_expert=[8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 8191, 0, 0, 1, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8179, 0, 0, 2, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8189, 0, 0, 8022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8190, 1, 0] payload_rows_per_dst=[16362, 16384, 16366, 16364, 16384, 16044, 16384, 16382] dup_rows_per_dst=[0, 324, 0, 0, 14, 0, 54, 10] combine_rows_per_src=[16362, 16384, 16366, 16364, 16384, 16044, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130850 payload_rows_remote=114466 dup_rows=222 dup_rows_remote=202 dedup_ratio=0.001694 router_tokens_per_expert=[8158, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 1, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16342, 16384, 16384, 16384, 16384, 16206, 16382, 16384] dup_rows_per_dst=[2, 122, 0, 20, 2, 0, 74, 2] combine_rows_per_src=[16342, 16384, 16384, 16384, 16384, 16206, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=938016768 dispatch_meta_bytes=459080 combine_payload_bytes=938016768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938475848 ingress_bytes=1013514240 link_bytes=1013514240 token_bytes=1876492616 weight_movement_bytes=75497472 total_nvlink_bytes=1951990088 weight_to_token_ratio=0.040233 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.038603 gemm_active_groups=13 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285278719e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=934854656 dispatch_meta_bytes=457240 combine_payload_bytes=934854656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935311896 ingress_bytes=1010352128 link_bytes=1010352128 token_bytes=1870166552 weight_movement_bytes=75497472 total_nvlink_bytes=1945664024 weight_to_token_ratio=0.040369 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281268716e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130150 payload_rows_remote=113768 dup_rows=922 dup_rows_remote=914 dedup_ratio=0.007034 router_tokens_per_expert=[8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16382, 16378, 16384, 16384, 15470, 16384, 16384] dup_rows_per_dst=[2, 8, 0, 6, 0, 0, 906, 0] combine_rows_per_src=[16384, 16382, 16378, 16384, 16384, 15470, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=936230912 dispatch_meta_bytes=458696 combine_payload_bytes=936230912 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=936689608 ingress_bytes=1087225856 link_bytes=1087225856 token_bytes=1872920520 weight_movement_bytes=150994944 total_nvlink_bytes=2023915464 weight_to_token_ratio=0.080620 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378755424e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=880066560 dispatch_meta_bytes=457288 combine_payload_bytes=880066560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=880523848 ingress_bytes=955564032 link_bytes=955564032 token_bytes=1760590408 weight_movement_bytes=75497472 total_nvlink_bytes=1836087880 weight_to_token_ratio=0.042882 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.211789698e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130382 payload_rows_remote=114000 dup_rows=690 dup_rows_remote=690 dedup_ratio=0.005264 router_tokens_per_expert=[8189, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 1, 0, 1, 1, 0, 0, 0, 8187, 8191, 0, 1, 0, 0, 0, 0, 8189, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16380, 15846, 16384, 16384, 16382, 16240, 16384, 16382] dup_rows_per_dst=[2, 138, 6, 6, 24, 514, 0, 0] combine_rows_per_src=[16380, 15846, 16384, 16384, 16382, 16240, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 2, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130838 payload_rows_remote=114582 dup_rows=234 dup_rows_remote=206 dedup_ratio=0.001785 router_tokens_per_expert=[8167, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 86, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 8187, 0, 0, 1, 1, 0, 0, 8161, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 1, 0, 0, 3, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8120, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 8190, 0, 0] payload_rows_per_dst=[16334, 16384, 16342, 16382, 16374, 16256, 16384, 16382] dup_rows_per_dst=[28, 74, 0, 18, 2, 28, 68, 16] combine_rows_per_src=[16334, 16384, 16342, 16382, 16374, 16256, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=937705472 dispatch_meta_bytes=458672 combine_payload_bytes=937705472 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=938164144 ingress_bytes=1013202944 link_bytes=1013202944 token_bytes=1875869616 weight_movement_bytes=75497472 total_nvlink_bytes=1951367088 weight_to_token_ratio=0.040247 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.038603 gemm_active_groups=13 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284883952e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=931987456 dispatch_meta_bytes=458728 combine_payload_bytes=931987456 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=932446184 ingress_bytes=1309474816 link_bytes=1309474816 token_bytes=1864433640 weight_movement_bytes=377487360 total_nvlink_bytes=2241921000 weight_to_token_ratio=0.202468 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.660598389e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=933888000 dispatch_meta_bytes=458760 combine_payload_bytes=933888000 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=934346760 ingress_bytes=1009385472 link_bytes=1009385472 token_bytes=1868234760 weight_movement_bytes=75497472 total_nvlink_bytes=1943732232 weight_to_token_ratio=0.040411 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.280042860e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=938655744 dispatch_meta_bytes=459152 combine_payload_bytes=938655744 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939114896 ingress_bytes=1089650688 link_bytes=1089650688 token_bytes=1877770640 weight_movement_bytes=150994944 total_nvlink_bytes=2028765584 weight_to_token_ratio=0.080412 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=64512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381830452e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130024 payload_rows_remote=114346 dup_rows=1048 dup_rows_remote=1048 dedup_ratio=0.007996 router_tokens_per_expert=[0, 5, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8180, 0, 0, 0, 8175, 12, 0, 0, 1, 8154, 0, 8145, 0, 0, 1, 10, 3, 0, 0, 0, 0, 0, 15, 0, 4, 0, 0, 14, 0, 8082, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 2, 194, 0, 0, 0, 0, 4, 0, 0, 138, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 9, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7826, 1, 0, 0, 1, 0, 84, 8168, 0, 0, 0, 0, 2, 0, 66, 8141, 0, 9] payload_rows_per_dst=[16376, 16380, 16198, 16332, 16362, 16320, 15678, 16378] dup_rows_per_dst=[74, 24, 32, 386, 284, 20, 0, 228] combine_rows_per_src=[16376, 16380, 16198, 16332, 16362, 16320, 15678, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125990 payload_rows_remote=109606 dup_rows=5082 dup_rows_remote=3942 dedup_ratio=0.038773 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8037, 0, 0, 0, 8109, 0, 0, 0, 0, 8102, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6301, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 1964, 0, 0, 0, 0, 5, 0, 0, 553, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 8191, 82, 1, 0, 0, 0, 0, 3, 8189, 0, 0] payload_rows_per_dst=[16076, 16220, 12602, 16366, 16384, 16382, 15578, 16382] dup_rows_per_dst=[0, 0, 0, 3772, 1140, 2, 0, 168] combine_rows_per_src=[16076, 16220, 12602, 16366, 16384, 16382, 15578, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125606 payload_rows_remote=111188 dup_rows=5466 dup_rows_remote=5466 dedup_ratio=0.041702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7762, 0, 0, 0, 8189, 0, 0, 0, 0, 8192, 0, 8190, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7209, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2083, 0, 0, 0, 0, 42, 0, 0, 578, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 6880, 0, 0, 0, 0, 0, 0, 8191, 15, 0, 0, 0, 0, 0, 0, 8174, 0, 0] payload_rows_per_dst=[15526, 16384, 14418, 16384, 16382, 16380, 13764, 16368] dup_rows_per_dst=[0, 46, 0, 4168, 1242, 0, 0, 10] combine_rows_per_src=[15526, 16384, 14418, 16384, 16382, 16380, 13764, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129078 payload_rows_remote=112702 dup_rows=1994 dup_rows_remote=1648 dedup_ratio=0.015213 router_tokens_per_expert=[8156, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 37, 0, 1, 0, 0, 0, 0, 0, 0, 8186, 8186, 0, 0, 0, 0, 0, 3, 8186, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8190, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 8151, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 7, 0, 4, 0, 1, 0, 1, 0, 1, 3, 0, 8190, 0, 0] payload_rows_per_dst=[16376, 16134, 16384, 16328, 16374, 16380, 14848, 16254] dup_rows_per_dst=[346, 70, 82, 0, 0, 1480, 0, 16] combine_rows_per_src=[16376, 16134, 16384, 16328, 16374, 16380, 14848, 16254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127300 payload_rows_remote=111152 dup_rows=3772 dup_rows_remote=3772 dedup_ratio=0.028778 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7828, 0, 0, 0, 7950, 0, 0, 0, 0, 8051, 0, 8192, 124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 637, 0, 0, 0, 0, 5, 0, 0, 1290, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[15656, 16148, 14328, 16196, 16384, 16384, 15820, 16384] dup_rows_per_dst=[0, 0, 0, 1180, 2590, 0, 0, 2] combine_rows_per_src=[15656, 16148, 14328, 16196, 16384, 16384, 15820, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=936722432 dispatch_meta_bytes=461576 combine_payload_bytes=936722432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937184008 ingress_bytes=1012219904 link_bytes=1012219904 token_bytes=1873906440 weight_movement_bytes=75497472 total_nvlink_bytes=1949403912 weight_to_token_ratio=0.040289 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=62976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283637319e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=897892352 dispatch_meta_bytes=454192 combine_payload_bytes=897892352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=898346544 ingress_bytes=973389824 link_bytes=973389824 token_bytes=1796238896 weight_movement_bytes=75497472 total_nvlink_bytes=1871736368 weight_to_token_ratio=0.042031 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=61184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.234395312e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130220 payload_rows_remote=113838 dup_rows=852 dup_rows_remote=410 dedup_ratio=0.006500 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8145, 0, 0, 0, 8179, 0, 0, 0, 0, 8190, 0, 8182, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 218, 0, 0, 0, 0, 34, 0, 0, 161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16294, 16376, 15760, 16382, 16384, 16366, 16274, 16384] dup_rows_per_dst=[0, 22, 0, 442, 388, 0, 0, 0] combine_rows_per_src=[16294, 16376, 15760, 16382, 16384, 16366, 16274, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=910852096 dispatch_meta_bytes=466616 combine_payload_bytes=910852096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=911318712 ingress_bytes=986349568 link_bytes=986349568 token_bytes=1822170808 weight_movement_bytes=75497472 total_nvlink_bytes=1897668280 weight_to_token_ratio=0.041433 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=58496 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.250830091e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=923254784 dispatch_meta_bytes=457400 combine_payload_bytes=923254784 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=923712184 ingress_bytes=998752256 link_bytes=998752256 token_bytes=1846966968 weight_movement_bytes=75497472 total_nvlink_bytes=1922464440 weight_to_token_ratio=0.040876 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.266558446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=106674 payload_rows_remote=90290 dup_rows=24398 dup_rows_remote=15564 dedup_ratio=0.186142 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 8190, 0, 0, 0, 8176, 0, 0, 0, 0, 8175, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7453, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 590, 0, 0, 0, 0, 53, 0, 0, 126, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 1, 0, 3, 8191, 0, 0, 0, 0, 0, 0, 2, 8191, 0, 0] payload_rows_per_dst=[16382, 2236, 14990, 16382, 7564, 16376, 16360, 16384] dup_rows_per_dst=[6, 0, 13974, 1160, 356, 68, 0, 8834] combine_rows_per_src=[16382, 2236, 14990, 16382, 7564, 16376, 16360, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128724 payload_rows_remote=112342 dup_rows=2348 dup_rows_remote=2334 dedup_ratio=0.017914 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 7948, 0, 0, 0, 8096, 0, 0, 0, 0, 8187, 0, 8188, 8, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 7433, 7, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 278, 0, 0, 0, 0, 108, 0, 0, 700, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 8057, 0, 0, 0, 0, 0, 7, 8192, 73, 0, 0, 0, 0, 4, 0, 8181, 0, 0] payload_rows_per_dst=[15910, 16208, 14876, 16376, 16384, 16382, 16204, 16384] dup_rows_per_dst=[0, 0, 14, 556, 1618, 14, 0, 146] combine_rows_per_src=[15910, 16208, 14876, 16376, 16384, 16382, 16204, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=910557184 dispatch_meta_bytes=459696 combine_payload_bytes=910557184 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=911016880 ingress_bytes=1288044544 link_bytes=1288044544 token_bytes=1821574064 weight_movement_bytes=377487360 total_nvlink_bytes=2199061424 weight_to_token_ratio=0.207231 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=58112 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.633421788e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=932560896 dispatch_meta_bytes=456992 combine_payload_bytes=932560896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933017888 ingress_bytes=1008058368 link_bytes=1008058368 token_bytes=1865578784 weight_movement_bytes=75497472 total_nvlink_bytes=1941076256 weight_to_token_ratio=0.040469 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=58752 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278359906e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=739655680 dispatch_meta_bytes=423416 combine_payload_bytes=739655680 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=740079096 ingress_bytes=1041645568 link_bytes=1041645568 token_bytes=1479734776 weight_movement_bytes=301989888 total_nvlink_bytes=1781724664 weight_to_token_ratio=0.204084 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=60800 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.320953203e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=920305664 dispatch_meta_bytes=458704 combine_payload_bytes=920305664 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=920764368 ingress_bytes=1071300608 link_bytes=1071300608 token_bytes=1841070032 weight_movement_bytes=150994944 total_nvlink_bytes=1992064976 weight_to_token_ratio=0.082015 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.358559968e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120460 payload_rows_remote=104082 dup_rows=10612 dup_rows_remote=606 dedup_ratio=0.080963 router_tokens_per_expert=[0, 7, 0, 5, 0, 8158, 0, 16, 0, 20, 0, 0, 0, 11, 8155, 0, 8166, 8010, 0, 11, 0, 0, 5, 0, 7, 22, 2, 0, 0, 0, 0, 0, 0, 0, 89, 17, 0, 0, 0, 0, 1, 0, 0, 0, 8, 0, 0, 0, 0, 14, 0, 0, 8171, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 22, 7783, 0, 0, 0, 0, 0, 0, 3263, 7, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 9, 5395, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 3] payload_rows_per_dst=[13064, 16092, 16384, 16372, 9550, 16326, 16378, 16294] dup_rows_per_dst=[42, 22, 178, 4, 358, 2, 10006, 0] combine_rows_per_src=[13064, 16092, 16384, 16372, 9550, 16326, 16378, 16294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119722 payload_rows_remote=108938 dup_rows=11350 dup_rows_remote=8086 dedup_ratio=0.086594 router_tokens_per_expert=[0, 0, 2, 0, 1, 8183, 0, 0, 0, 0, 0, 0, 0, 1245, 8191, 0, 8191, 8139, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 49, 0, 1756, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 5263, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8189, 2, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16286, 16384, 16384, 10784, 16374, 10742, 16384] dup_rows_per_dst=[2494, 0, 12, 0, 3264, 2, 5574, 4] combine_rows_per_src=[16384, 16286, 16384, 16384, 10784, 16374, 10742, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124510 payload_rows_remote=108126 dup_rows=6562 dup_rows_remote=6558 dedup_ratio=0.050064 router_tokens_per_expert=[0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 741, 8180, 0, 8192, 8166, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2511, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4916, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16368, 9838, 16384] dup_rows_per_dst=[1458, 76, 4, 2, 5022, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16368, 9838, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=107692 payload_rows_remote=91308 dup_rows=23380 dup_rows_remote=20992 dedup_ratio=0.178375 router_tokens_per_expert=[0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8065, 0, 0, 0, 8186, 0, 0, 0, 0, 8175, 0, 8187, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 6938, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 817, 0, 0, 0, 0, 0, 0, 1, 612, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 2, 8188, 4, 0, 0, 1, 0, 0, 3, 8191, 0, 0] payload_rows_per_dst=[16384, 16382, 13880, 4988, 16384, 16376, 16284, 7014] dup_rows_per_dst=[2388, 14646, 0, 2, 1226, 2, 5110, 6] combine_rows_per_src=[16384, 16382, 13880, 4988, 16384, 16376, 16284, 7014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=852639744 dispatch_meta_bytes=418752 combine_payload_bytes=852639744 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=853058496 ingress_bytes=928137216 link_bytes=928137216 token_bytes=1705698240 weight_movement_bytes=75497472 total_nvlink_bytes=1781195712 weight_to_token_ratio=0.044262 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=38528 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177008635e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=892420096 dispatch_meta_bytes=468096 combine_payload_bytes=892420096 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=892888192 ingress_bytes=1043415040 link_bytes=1043415040 token_bytes=1785308288 weight_movement_bytes=150994944 total_nvlink_bytes=1936303232 weight_to_token_ratio=0.084576 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=38016 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.323197142e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124020 payload_rows_remote=107636 dup_rows=7052 dup_rows_remote=7050 dedup_ratio=0.053802 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 330, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 1, 0, 3220, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 4664, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 9336, 16384] dup_rows_per_dst=[660, 2, 0, 0, 6388, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 9336, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119076 payload_rows_remote=102692 dup_rows=11996 dup_rows_remote=11996 dedup_ratio=0.091522 router_tokens_per_expert=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 4732, 8192, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 10, 0, 1235, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 0, 0, 2192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16380, 16384, 4396, 16384] dup_rows_per_dst=[9466, 30, 2, 0, 2486, 10, 0, 2] combine_rows_per_src=[16384, 16380, 16384, 16384, 16380, 16384, 4396, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=885768192 dispatch_meta_bytes=458736 combine_payload_bytes=885768192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=886226928 ingress_bytes=961265664 link_bytes=961265664 token_bytes=1771995120 weight_movement_bytes=75497472 total_nvlink_bytes=1847492592 weight_to_token_ratio=0.042606 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.219020170e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116464 payload_rows_remote=100080 dup_rows=14608 dup_rows_remote=14602 dedup_ratio=0.111450 router_tokens_per_expert=[0, 1, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 949, 8191, 0, 8189, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 1, 0, 4248, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 14, 0, 0, 2986, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 0, 0] payload_rows_per_dst=[1900, 16382, 16380, 16376, 16276, 16384, 16382, 16384] dup_rows_per_dst=[2, 8, 0, 2, 8456, 154, 5980, 6] combine_rows_per_src=[1900, 16382, 16380, 16376, 16276, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=747995136 dispatch_meta_bytes=449200 combine_payload_bytes=747995136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=748444336 ingress_bytes=823492608 link_bytes=823492608 token_bytes=1496439472 weight_movement_bytes=75497472 total_nvlink_bytes=1571936944 weight_to_token_ratio=0.050451 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.044304542e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114838 payload_rows_remote=98494 dup_rows=16234 dup_rows_remote=16234 dedup_ratio=0.123856 router_tokens_per_expert=[0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 2, 0, 1792, 8154, 0, 8191, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 51, 0, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 6307, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 22, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16362, 16368, 16382, 16384, 234, 16344, 16380, 16384] dup_rows_per_dst=[3534, 0, 0, 20, 0, 0, 12618, 62] combine_rows_per_src=[16362, 16368, 16382, 16384, 234, 16344, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=881754112 dispatch_meta_bytes=458744 combine_payload_bytes=881754112 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=882212856 ingress_bytes=1108246528 link_bytes=1108246528 token_bytes=1763966968 weight_movement_bytes=226492416 total_nvlink_bytes=1990459384 weight_to_token_ratio=0.128399 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.405412595e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=841252864 dispatch_meta_bytes=458752 combine_payload_bytes=841252864 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=841711616 ingress_bytes=992247808 link_bytes=992247808 token_bytes=1682964480 weight_movement_bytes=150994944 total_nvlink_bytes=1833959424 weight_to_token_ratio=0.089720 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.258309890e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=819855360 dispatch_meta_bytes=458728 combine_payload_bytes=819855360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=820314088 ingress_bytes=895352832 link_bytes=895352832 token_bytes=1640169448 weight_movement_bytes=75497472 total_nvlink_bytes=1715666920 weight_to_token_ratio=0.046030 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.135433421e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114530 payload_rows_remote=98174 dup_rows=16542 dup_rows_remote=5994 dedup_ratio=0.126205 router_tokens_per_expert=[0, 0, 0, 0, 0, 8166, 0, 6, 0, 14, 0, 0, 0, 8165, 0, 0, 8, 0, 0, 0, 1, 8176, 0, 0, 0, 0, 4, 1, 8133, 1, 0, 0, 0, 0, 0, 8160, 0, 0, 7498, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 3, 0, 59, 0, 0, 0, 0, 0, 0, 3514, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8, 0, 0, 41, 17, 0, 3, 14, 0, 0, 0, 0, 1, 0, 16, 5272, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 10, 1, 0, 13, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16378, 20, 16372, 16350, 16384, 16356, 16306] dup_rows_per_dst=[6, 4, 0, 5818, 10, 148, 10548, 8] combine_rows_per_src=[16364, 16378, 20, 16372, 16350, 16384, 16356, 16306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124216 payload_rows_remote=107832 dup_rows=6856 dup_rows_remote=6842 dedup_ratio=0.052307 router_tokens_per_expert=[0, 0, 4, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8156, 0, 2, 8070, 0, 0, 0, 0, 0, 0, 0, 81, 0, 0, 0, 0, 0, 1, 0, 1, 2, 0, 0, 4909, 0, 0, 26, 0, 0, 0, 2, 0, 0, 0, 0, 8191, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 3290, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0] payload_rows_per_dst=[16266, 16360, 16306, 9818, 16384, 16384, 16314, 16384] dup_rows_per_dst=[4, 0, 0, 60, 14, 176, 6580, 22] combine_rows_per_src=[16266, 16360, 16306, 9818, 16384, 16384, 16314, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=806862848 dispatch_meta_bytes=458912 combine_payload_bytes=806862848 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=807321760 ingress_bytes=957857792 link_bytes=957857792 token_bytes=1614184608 weight_movement_bytes=150994944 total_nvlink_bytes=1765179552 weight_to_token_ratio=0.093543 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.214698509e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126468 payload_rows_remote=110090 dup_rows=4604 dup_rows_remote=1826 dedup_ratio=0.035126 router_tokens_per_expert=[0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 12, 8, 0, 8180, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 98, 7506, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 686, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[16112, 16318, 16378, 15014, 16384, 16384, 13494, 16384] dup_rows_per_dst=[0, 0, 2778, 232, 4, 212, 1366, 12] combine_rows_per_src=[16112, 16318, 16378, 15014, 16384, 16384, 13494, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127874 payload_rows_remote=111500 dup_rows=3198 dup_rows_remote=3018 dedup_ratio=0.024399 router_tokens_per_expert=[0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 99, 8176, 0, 8190, 8191, 0, 0, 0, 2, 1, 0, 1, 0, 0, 0, 0, 38, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 1, 0, 1362, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6717, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 5, 0] payload_rows_per_dst=[16374, 16382, 16194, 16376, 16384, 16338, 13442, 16384] dup_rows_per_dst=[180, 276, 0, 0, 2726, 2, 0, 14] combine_rows_per_src=[16374, 16382, 16194, 16376, 16384, 16338, 13442, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=804241408 dispatch_meta_bytes=416672 combine_payload_bytes=804241408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=804658080 ingress_bytes=879738880 link_bytes=879738880 token_bytes=1608899488 weight_movement_bytes=75497472 total_nvlink_bytes=1684396960 weight_to_token_ratio=0.046925 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=47616 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.115632732e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=883359744 dispatch_meta_bytes=458696 combine_payload_bytes=883359744 prefetch_bytes=0 reduce_bytes=0 egress_bytes=883818440 ingress_bytes=883359744 link_bytes=883818440 token_bytes=1767178184 weight_movement_bytes=0 total_nvlink_bytes=1767178184 weight_to_token_ratio=0.000000 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.120806189e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123004 payload_rows_remote=114572 dup_rows=8068 dup_rows_remote=8038 dedup_ratio=0.061554 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 53, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 6, 0, 10, 1, 0, 0, 4214, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 3969, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16298, 16354, 16384, 8432, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 84, 30, 0, 16, 7938, 0] combine_rows_per_src=[16298, 16354, 16384, 8432, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115298 payload_rows_remote=98914 dup_rows=15774 dup_rows_remote=15774 dedup_ratio=0.120346 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7848, 0, 0, 0, 0, 0, 1, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7954, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16316, 16384, 16384, 15908, 16384, 16384, 1154, 16384] dup_rows_per_dst=[0, 0, 15748, 0, 0, 26, 0, 0] combine_rows_per_src=[16316, 16384, 16384, 15908, 16384, 16384, 1154, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=901857280 dispatch_meta_bytes=447664 combine_payload_bytes=901857280 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=902304944 ingress_bytes=1203847168 link_bytes=1203847168 token_bytes=1804162224 weight_movement_bytes=301989888 total_nvlink_bytes=2106152112 weight_to_token_ratio=0.167385 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=47232 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.526647663e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114648 payload_rows_remote=98268 dup_rows=16424 dup_rows_remote=16386 dedup_ratio=0.125305 router_tokens_per_expert=[0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 8186, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 8186, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 4927, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3254, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16346, 22, 16382, 16384, 16380, 16380, 16380] dup_rows_per_dst=[2, 0, 4, 9864, 2, 2, 6512, 38] combine_rows_per_src=[16374, 16346, 22, 16382, 16384, 16380, 16380, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=913408000 dispatch_meta_bytes=458072 combine_payload_bytes=913408000 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=913866072 ingress_bytes=1215397888 link_bytes=1215397888 token_bytes=1827274072 weight_movement_bytes=301989888 total_nvlink_bytes=2129263960 weight_to_token_ratio=0.165268 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=55680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.541295602e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128524 payload_rows_remote=112142 dup_rows=2548 dup_rows_remote=2548 dedup_ratio=0.019440 router_tokens_per_expert=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 8, 1, 0, 8143, 121, 11, 8110, 0, 0, 0, 0, 0, 1, 0, 115, 1, 0, 0, 0, 0, 70, 0, 33, 0, 0, 30, 7958, 0, 0, 54, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16256, 16060, 14554, 16218, 16384, 16382, 16286, 16384] dup_rows_per_dst=[0, 0, 352, 1902, 0, 0, 294, 0] combine_rows_per_src=[16256, 16060, 14554, 16218, 16384, 16382, 16286, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=938573824 dispatch_meta_bytes=490440 combine_payload_bytes=938573824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939064264 ingress_bytes=1014071296 link_bytes=1014071296 token_bytes=1877638088 weight_movement_bytes=75497472 total_nvlink_bytes=1953135560 weight_to_token_ratio=0.040209 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=48512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285985145e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=810303488 dispatch_meta_bytes=458752 combine_payload_bytes=810303488 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=810762240 ingress_bytes=1036795904 link_bytes=1036795904 token_bytes=1621065728 weight_movement_bytes=226492416 total_nvlink_bytes=1847558144 weight_to_token_ratio=0.139718 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.314803146e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=805011456 dispatch_meta_bytes=458616 combine_payload_bytes=805011456 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=805470072 ingress_bytes=956006400 link_bytes=956006400 token_bytes=1610481528 weight_movement_bytes=150994944 total_nvlink_bytes=1761476472 weight_to_token_ratio=0.093758 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212350683e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126312 payload_rows_remote=109980 dup_rows=4760 dup_rows_remote=4744 dedup_ratio=0.036316 router_tokens_per_expert=[0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 28, 0, 0, 0, 0, 37, 0, 0, 7556, 0, 35, 0, 301, 0, 1, 7, 0, 0, 14, 0, 0, 6265, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8, 22, 8, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8165, 2, 0, 0, 0, 22, 4, 0, 0, 0, 2128, 0, 0, 0, 8160, 0, 5, 0, 0, 0, 0, 0, 0, 117, 8170, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16338, 15706, 12558, 16368, 16356, 16368, 16332, 16286] dup_rows_per_dst=[4, 168, 0, 66, 4286, 220, 16, 0] combine_rows_per_src=[16338, 15706, 12558, 16368, 16356, 16368, 16332, 16286] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=918667264 dispatch_meta_bytes=458760 combine_payload_bytes=918667264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919126024 ingress_bytes=994164736 link_bytes=994164736 token_bytes=1837793288 weight_movement_bytes=75497472 total_nvlink_bytes=1913290760 weight_to_token_ratio=0.041081 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.260740825e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129690 payload_rows_remote=113306 dup_rows=1382 dup_rows_remote=38 dedup_ratio=0.010544 router_tokens_per_expert=[0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 75, 7665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7975, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8105, 0, 0, 4, 0, 0, 0, 0, 0, 0, 755, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6, 1, 0, 0, 0, 1] payload_rows_per_dst=[16382, 15480, 15950, 16384, 16384, 16384, 16356, 16370] dup_rows_per_dst=[10, 0, 0, 16, 1344, 0, 0, 12] combine_rows_per_src=[16382, 15480, 15950, 16384, 16384, 16384, 16356, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130426 payload_rows_remote=114086 dup_rows=646 dup_rows_remote=642 dedup_ratio=0.004929 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7908, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16] payload_rows_per_dst=[16384, 15826, 16340, 16384, 16384, 16384, 16342, 16382] dup_rows_per_dst=[16, 0, 4, 4, 592, 0, 0, 30] combine_rows_per_src=[16384, 15826, 16340, 16384, 16384, 16384, 16342, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114076 payload_rows_remote=97694 dup_rows=16996 dup_rows_remote=16304 dedup_ratio=0.129669 router_tokens_per_expert=[0, 0, 0, 0, 0, 8189, 0, 1, 2, 0, 0, 0, 0, 8133, 1, 0, 1, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 17, 8135, 0, 0, 0, 1, 0, 1, 0, 23, 0, 0, 0, 3, 0, 29, 0, 0, 0, 0, 33, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 20, 0, 0, 0, 1, 0, 7, 0, 0, 1, 1, 731, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16360, 16384, 14912, 16382, 15806, 1466, 16384] dup_rows_per_dst=[692, 508, 15674, 110, 6, 4, 0, 2] combine_rows_per_src=[16382, 16360, 16384, 14912, 16382, 15806, 1466, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=900956160 dispatch_meta_bytes=458896 combine_payload_bytes=900956160 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=901415056 ingress_bytes=976453632 link_bytes=976453632 token_bytes=1802371216 weight_movement_bytes=75497472 total_nvlink_bytes=1877868688 weight_to_token_ratio=0.041888 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.238280652e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=928202752 dispatch_meta_bytes=453376 combine_payload_bytes=928202752 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=928656128 ingress_bytes=1305690112 link_bytes=1305690112 token_bytes=1856858880 weight_movement_bytes=377487360 total_nvlink_bytes=2234346240 weight_to_token_ratio=0.203294 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=53888 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.655798852e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125750 payload_rows_remote=109366 dup_rows=5322 dup_rows_remote=5322 dedup_ratio=0.040604 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 8064, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5647, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2671, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 4] payload_rows_per_dst=[16384, 16156, 11294, 16384, 16384, 16384, 16382, 16382] dup_rows_per_dst=[0, 0, 2, 0, 5310, 0, 0, 10] combine_rows_per_src=[16384, 16156, 11294, 16384, 16384, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125064 payload_rows_remote=109222 dup_rows=6008 dup_rows_remote=6008 dedup_ratio=0.045837 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 7895, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 333, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17] payload_rows_per_dst=[16384, 15842, 16380, 16384, 10930, 16384, 16378, 16382] dup_rows_per_dst=[0, 0, 5970, 0, 4, 0, 0, 34] combine_rows_per_src=[16384, 15842, 16380, 16384, 10930, 16384, 16378, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=934592512 dispatch_meta_bytes=458912 combine_payload_bytes=934592512 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=935051424 ingress_bytes=1085587456 link_bytes=1085587456 token_bytes=1869643936 weight_movement_bytes=150994944 total_nvlink_bytes=2020638880 weight_to_token_ratio=0.080761 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=56704 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.376677702e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125376 payload_rows_remote=108992 dup_rows=5696 dup_rows_remote=5692 dedup_ratio=0.043457 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 26, 7988, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 5703, 0, 3, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2689, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[16380, 16032, 11406, 16316, 16330, 16170, 16358, 16384] dup_rows_per_dst=[0, 2, 6, 0, 5348, 6, 330, 4] combine_rows_per_src=[16380, 16032, 11406, 16316, 16330, 16170, 16358, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=800309248 dispatch_meta_bytes=455992 combine_payload_bytes=800309248 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=800765240 ingress_bytes=951304192 link_bytes=951304192 token_bytes=1601074488 weight_movement_bytes=150994944 total_nvlink_bytes=1752069432 weight_to_token_ratio=0.094309 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.206387622e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=895926272 dispatch_meta_bytes=458752 combine_payload_bytes=895926272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=896385024 ingress_bytes=971423744 link_bytes=971423744 token_bytes=1792311296 weight_movement_bytes=75497472 total_nvlink_bytes=1867808768 weight_to_token_ratio=0.042123 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.231902046e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128156 payload_rows_remote=111776 dup_rows=2916 dup_rows_remote=2916 dedup_ratio=0.022247 router_tokens_per_expert=[0, 0, 0, 0, 0, 12, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 115, 7686, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7139, 0, 37, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1525, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15604, 14336, 16382, 16384, 16380, 16302, 16384] dup_rows_per_dst=[32, 48, 18, 2, 2816, 0, 0, 0] combine_rows_per_src=[16384, 15604, 14336, 16382, 16384, 16380, 16302, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=894746624 dispatch_meta_bytes=460920 combine_payload_bytes=894746624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=895207544 ingress_bytes=970244096 link_bytes=970244096 token_bytes=1789954168 weight_movement_bytes=75497472 total_nvlink_bytes=1865451640 weight_to_token_ratio=0.042178 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=62464 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.230406086e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=892862464 dispatch_meta_bytes=458736 combine_payload_bytes=892862464 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=893321200 ingress_bytes=968359936 link_bytes=968359936 token_bytes=1786183664 weight_movement_bytes=75497472 total_nvlink_bytes=1861681136 weight_to_token_ratio=0.042267 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.228016706e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109658 payload_rows_remote=97530 dup_rows=21414 dup_rows_remote=21234 dedup_ratio=0.163376 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 1415, 0, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 162, 0, 6, 0, 0, 0, 3, 6, 8, 0, 44, 0, 11, 0, 6, 0, 6, 0, 6, 1, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 15, 0, 8168, 0, 0, 7867, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 76, 0, 5520, 21, 0, 0, 0, 537, 0, 0, 0, 0, 0, 995, 1, 0, 0, 8141, 0, 9, 0, 8177, 0, 0, 8163, 0] payload_rows_per_dst=[15872, 16320, 364, 16384, 16346, 15868, 12128, 16376] dup_rows_per_dst=[2820, 0, 6, 118, 0, 20, 180, 18270] combine_rows_per_src=[15872, 16320, 364, 16384, 16346, 15868, 12128, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130498 payload_rows_remote=114114 dup_rows=574 dup_rows_remote=540 dedup_ratio=0.004379 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 171, 8188, 1, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 8032, 3, 0, 0, 0, 8192, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 78, 0, 0, 8185, 0] payload_rows_per_dst=[16286, 16384, 16228, 16378, 16384, 16070, 16384, 16384] dup_rows_per_dst=[6, 374, 0, 2, 34, 0, 2, 156] combine_rows_per_src=[16286, 16384, 16228, 16378, 16384, 16070, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=915668992 dispatch_meta_bytes=458768 combine_payload_bytes=915668992 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=916127760 ingress_bytes=1066663936 link_bytes=1066663936 token_bytes=1831796752 weight_movement_bytes=150994944 total_nvlink_bytes=1982791696 weight_to_token_ratio=0.082430 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.352680015e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130836 payload_rows_remote=114614 dup_rows=236 dup_rows_remote=234 dedup_ratio=0.001801 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 8189, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 8191, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 69, 0, 0, 8192, 0] payload_rows_per_dst=[16334, 16384, 16222, 16384, 16384, 16360, 16384, 16384] dup_rows_per_dst=[24, 70, 2, 2, 0, 0, 0, 138] combine_rows_per_src=[16334, 16384, 16222, 16384, 16384, 16360, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116144 payload_rows_remote=99770 dup_rows=14928 dup_rows_remote=14928 dedup_ratio=0.113892 router_tokens_per_expert=[0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 7651, 0, 0, 0, 0, 2, 1, 0, 3, 0, 1, 0, 0, 7604, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1109, 0, 0, 0, 8189, 0, 0, 0, 0, 3, 0, 0, 0, 1, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 57] payload_rows_per_dst=[16374, 16384, 15766, 16380, 2220, 16384, 16276, 16360] dup_rows_per_dst=[0, 3824, 11042, 2, 0, 8, 0, 52] combine_rows_per_src=[16374, 16384, 15766, 16380, 2220, 16384, 16276, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=798965760 dispatch_meta_bytes=475056 combine_payload_bytes=798965760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=799440816 ingress_bytes=874463232 link_bytes=874463232 token_bytes=1598406576 weight_movement_bytes=75497472 total_nvlink_bytes=1673904048 weight_to_token_ratio=0.047233 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=62976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.108942468e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=934821888 dispatch_meta_bytes=458616 combine_payload_bytes=934821888 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=935280504 ingress_bytes=1085816832 link_bytes=1085816832 token_bytes=1870102392 weight_movement_bytes=150994944 total_nvlink_bytes=2021097336 weight_to_token_ratio=0.080742 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.376968583e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130994 payload_rows_remote=114610 dup_rows=78 dup_rows_remote=62 dedup_ratio=0.000595 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 8192, 0] payload_rows_per_dst=[16382, 16384, 16362, 16384, 16384, 16330, 16384, 16384] dup_rows_per_dst=[18, 18, 2, 16, 2, 0, 0, 22] combine_rows_per_src=[16382, 16384, 16362, 16384, 16384, 16330, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124876 payload_rows_remote=108492 dup_rows=6196 dup_rows_remote=6192 dedup_ratio=0.047272 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 8192, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 8192, 0, 0, 0, 17, 0, 0, 8192, 0] payload_rows_per_dst=[16212, 16384, 16354, 16384, 16384, 16378, 16382, 10398] dup_rows_per_dst=[2, 4, 2104, 2, 0, 0, 4054, 30] combine_rows_per_src=[16212, 16384, 16354, 16384, 16384, 16378, 16382, 10398] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128036 payload_rows_remote=111652 dup_rows=3036 dup_rows_remote=1122 dedup_ratio=0.023163 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6, 8190, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8061, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 28, 8192, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8191, 1, 0, 0, 110, 0, 0, 8192, 0] payload_rows_per_dst=[14396, 15748, 16122, 16238, 16384, 16382, 16382, 16384] dup_rows_per_dst=[0, 14, 0, 2, 0, 1104, 2, 1914] combine_rows_per_src=[14396, 15748, 16122, 16238, 16384, 16382, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938917888 dispatch_meta_bytes=459392 combine_payload_bytes=938917888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939377280 ingress_bytes=1014415360 link_bytes=1014415360 token_bytes=1878295168 weight_movement_bytes=75497472 total_nvlink_bytes=1953792640 weight_to_token_ratio=0.040195 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=56320 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286421466e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=817315840 dispatch_meta_bytes=458792 combine_payload_bytes=817315840 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=817774632 ingress_bytes=892813312 link_bytes=892813312 token_bytes=1635090472 weight_movement_bytes=75497472 total_nvlink_bytes=1710587944 weight_to_token_ratio=0.046173 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.132212953e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130862 payload_rows_remote=114644 dup_rows=210 dup_rows_remote=210 dedup_ratio=0.001602 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8192, 0, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 0, 0, 8192, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16342, 16384, 16382, 16384, 16384, 16218, 16384, 16384] dup_rows_per_dst=[2, 192, 14, 0, 0, 0, 0, 2] combine_rows_per_src=[16342, 16384, 16382, 16384, 16384, 16218, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938885120 dispatch_meta_bytes=458688 combine_payload_bytes=938885120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939343808 ingress_bytes=1014382592 link_bytes=1014382592 token_bytes=1878228928 weight_movement_bytes=75497472 total_nvlink_bytes=1953726400 weight_to_token_ratio=0.040196 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286379912e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=888766464 dispatch_meta_bytes=458736 combine_payload_bytes=888766464 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=889225200 ingress_bytes=964263936 link_bytes=964263936 token_bytes=1777991664 weight_movement_bytes=75497472 total_nvlink_bytes=1853489136 weight_to_token_ratio=0.042462 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.222822401e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=914653184 dispatch_meta_bytes=451096 combine_payload_bytes=914653184 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=915104280 ingress_bytes=1216643072 link_bytes=1216643072 token_bytes=1829757464 weight_movement_bytes=301989888 total_nvlink_bytes=2131747352 weight_to_token_ratio=0.165044 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=54272 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.542874671e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 2, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116474 payload_rows_remote=100122 dup_rows=14598 dup_rows_remote=14128 dedup_ratio=0.111374 router_tokens_per_expert=[0, 3, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 8165, 0, 0, 0, 0, 53, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 12, 0, 8034, 74, 0, 1074, 0, 0, 39, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8086, 0, 6, 0, 16, 0, 0, 0, 6828, 0, 0, 1, 0, 0, 6, 0, 0, 8146, 0, 0, 0, 17, 0, 0, 0, 6, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 30, 0, 3, 8137, 0, 0, 0, 0, 225, 0, 0, 16, 13, 0, 0, 9, 0, 0, 8112, 0, 0, 0, 17, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16362, 16336, 2230, 16214, 16338, 16352, 16260] dup_rows_per_dst=[104, 106, 176, 4, 13546, 150, 470, 42] combine_rows_per_src=[16382, 16362, 16336, 2230, 16214, 16338, 16352, 16260] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125742 payload_rows_remote=109980 dup_rows=5330 dup_rows_remote=828 dedup_ratio=0.040665 router_tokens_per_expert=[37, 1, 8178, 0, 0, 166, 0, 0, 3, 0, 0, 0, 0, 0, 1, 8041, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 680, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 7509, 0, 0, 0, 1, 0, 0, 0, 0, 8129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16336, 16356, 16384, 11882, 15762, 16258, 16384, 16380] dup_rows_per_dst=[162, 0, 4, 654, 4502, 0, 8, 0] combine_rows_per_src=[16336, 16356, 16384, 11882, 15762, 16258, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939163648 dispatch_meta_bytes=459416 combine_payload_bytes=939163648 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=939623064 ingress_bytes=1316651008 link_bytes=1316651008 token_bytes=1878786712 weight_movement_bytes=377487360 total_nvlink_bytes=2256274072 weight_to_token_ratio=0.200921 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.669698811e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125292 payload_rows_remote=109074 dup_rows=5780 dup_rows_remote=5780 dedup_ratio=0.044098 router_tokens_per_expert=[31, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 0, 0, 2774, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 5474, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 95, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16270, 16326, 16218, 16384, 10948, 16378, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 5570, 0, 0, 210, 0] combine_rows_per_src=[16270, 16326, 16218, 16384, 10948, 16378, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=820199424 dispatch_meta_bytes=457000 combine_payload_bytes=820199424 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=820656424 ingress_bytes=1122189312 link_bytes=1122189312 token_bytes=1640855848 weight_movement_bytes=301989888 total_nvlink_bytes=1942845736 weight_to_token_ratio=0.184044 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.423094008e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=900956160 dispatch_meta_bytes=443232 combine_payload_bytes=900956160 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=901399392 ingress_bytes=1127448576 link_bytes=1127448576 token_bytes=1802355552 weight_movement_bytes=226492416 total_nvlink_bytes=2028847968 weight_to_token_ratio=0.125665 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=38528 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.429763495e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116470 payload_rows_remote=100094 dup_rows=14602 dup_rows_remote=14602 dedup_ratio=0.111404 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8187, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7263, 0, 1, 1, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 1, 0, 8189, 0, 0, 8188, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 8187, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 921, 1, 0, 8190, 0] payload_rows_per_dst=[16376, 16378, 16382, 16384, 16382, 16342, 16382, 1844] dup_rows_per_dst=[0, 34, 14528, 10, 4, 0, 24, 2] combine_rows_per_src=[16376, 16378, 16382, 16384, 16382, 16342, 16382, 1844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124396 payload_rows_remote=108012 dup_rows=6676 dup_rows_remote=376 dedup_ratio=0.050934 router_tokens_per_expert=[6, 0, 8190, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 3138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 5055, 0, 0, 0, 0, 0, 0, 0, 0, 8122, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16144, 16380, 16366, 16384, 10110, 16244, 16384, 16384] dup_rows_per_dst=[6, 0, 0, 6300, 0, 0, 370, 0] combine_rows_per_src=[16144, 16380, 16366, 16384, 10110, 16244, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121796 payload_rows_remote=105412 dup_rows=9276 dup_rows_remote=9276 dedup_ratio=0.070770 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 4362, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 3607, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 275, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16296, 16384, 16366, 16384, 7214, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 8724, 0, 0, 552, 0] combine_rows_per_src=[16296, 16384, 16366, 16384, 7214, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=893534208 dispatch_meta_bytes=459416 combine_payload_bytes=893534208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=893993624 ingress_bytes=969031680 link_bytes=969031680 token_bytes=1787527832 weight_movement_bytes=75497472 total_nvlink_bytes=1863025304 weight_to_token_ratio=0.042236 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.228868572e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128780 payload_rows_remote=112410 dup_rows=2292 dup_rows_remote=2180 dedup_ratio=0.017487 router_tokens_per_expert=[7, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7736, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 0, 8168, 7, 0, 7710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 918, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[15214, 16382, 16350, 15420, 16382, 16384, 16278, 16370] dup_rows_per_dst=[0, 274, 16, 0, 0, 1808, 82, 112] combine_rows_per_src=[15214, 16382, 16350, 15420, 16382, 16384, 16278, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=819970048 dispatch_meta_bytes=458784 combine_payload_bytes=819970048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=820428832 ingress_bytes=895467520 link_bytes=895467520 token_bytes=1640398880 weight_movement_bytes=75497472 total_nvlink_bytes=1715896352 weight_to_token_ratio=0.046024 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.135578862e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=884834304 dispatch_meta_bytes=433552 combine_payload_bytes=884834304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=885267856 ingress_bytes=960331776 link_bytes=960331776 token_bytes=1770102160 weight_movement_bytes=75497472 total_nvlink_bytes=1845599632 weight_to_token_ratio=0.042651 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=38400 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.217835869e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120492 payload_rows_remote=105620 dup_rows=10580 dup_rows_remote=10580 dedup_ratio=0.080719 router_tokens_per_expert=[36, 0, 8071, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7864, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 4079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 5100, 0, 0, 0, 0, 0, 0, 0, 0, 7436, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 168, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15798, 16142, 16380, 8174, 16384, 14872, 16384, 16358] dup_rows_per_dst=[26, 0, 2, 2, 10200, 0, 350, 0] combine_rows_per_src=[15798, 16142, 16380, 8174, 16384, 14872, 16384, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=863535104 dispatch_meta_bytes=458752 combine_payload_bytes=863535104 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=863993856 ingress_bytes=1014530048 link_bytes=1014530048 token_bytes=1727528960 weight_movement_bytes=150994944 total_nvlink_bytes=1878523904 weight_to_token_ratio=0.087405 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286566907e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=920862720 dispatch_meta_bytes=458360 combine_payload_bytes=920862720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=921321080 ingress_bytes=996360192 link_bytes=996360192 token_bytes=1842183800 weight_movement_bytes=75497472 total_nvlink_bytes=1917681272 weight_to_token_ratio=0.040983 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.263524972e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130496 payload_rows_remote=114370 dup_rows=576 dup_rows_remote=576 dedup_ratio=0.004395 router_tokens_per_expert=[1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1239, 0, 0, 6835, 0, 0, 0, 8182, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8063, 0, 0, 0, 0, 9, 0, 94, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8, 0, 0] payload_rows_per_dst=[16150, 16368, 16382, 16382, 16126, 16384, 16320, 16384] dup_rows_per_dst=[2, 2, 0, 2, 0, 200, 0, 370] combine_rows_per_src=[16150, 16368, 16382, 16382, 16126, 16384, 16320, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126128 payload_rows_remote=114036 dup_rows=4944 dup_rows_remote=4902 dedup_ratio=0.037720 router_tokens_per_expert=[0, 0, 0, 0, 56, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 8137, 245, 0, 0, 8164, 0, 0, 73, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8083, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 8160, 0, 11, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 0, 8083, 3, 0, 0, 0, 6, 7962, 9, 0, 0, 0, 2105, 4, 0, 0, 0, 0, 0, 0, 0, 8130, 1, 0, 0, 0, 0, 0, 6011, 0, 0, 2, 0, 28, 21, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 117, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 26] payload_rows_per_dst=[16362, 16384, 16208, 16342, 16036, 16322, 12092, 16382] dup_rows_per_dst=[30, 604, 0, 6, 26, 4158, 42, 78] combine_rows_per_src=[16362, 16384, 16208, 16342, 16036, 16322, 12092, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=865239040 dispatch_meta_bytes=464800 combine_payload_bytes=865239040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=865703840 ingress_bytes=940736512 link_bytes=940736512 token_bytes=1730942880 weight_movement_bytes=75497472 total_nvlink_bytes=1806440352 weight_to_token_ratio=0.043616 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.192986316e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129166 payload_rows_remote=112782 dup_rows=1906 dup_rows_remote=1906 dedup_ratio=0.014542 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 722, 0, 0, 6566, 0, 0, 0, 8176, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 251, 0, 6, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 188, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 488, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0] payload_rows_per_dst=[14576, 16362, 16384, 16384, 16322, 16384, 16370, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 512, 408, 984] combine_rows_per_src=[14576, 16362, 16384, 16384, 16322, 16384, 16370, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=936919040 dispatch_meta_bytes=459784 combine_payload_bytes=936919040 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=937378824 ingress_bytes=1163411456 link_bytes=1163411456 token_bytes=1874297864 weight_movement_bytes=226492416 total_nvlink_bytes=2100790280 weight_to_token_ratio=0.120841 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.475369489e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=934182912 dispatch_meta_bytes=475752 combine_payload_bytes=934182912 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=934658664 ingress_bytes=1009680384 link_bytes=1009680384 token_bytes=1868841576 weight_movement_bytes=75497472 total_nvlink_bytes=1944339048 weight_to_token_ratio=0.040398 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=62976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.280416850e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122352 payload_rows_remote=105976 dup_rows=8720 dup_rows_remote=6762 dedup_ratio=0.066528 router_tokens_per_expert=[0, 3, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 41, 0, 0, 3, 0, 0, 0, 0, 8125, 3, 0, 3291, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 1, 0, 0, 0, 0, 1, 4902, 0, 0, 1, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 8190, 0, 0, 36, 0, 50, 0, 0, 0, 0, 0, 0, 0, 8, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16360, 16334, 16380, 9806, 16256, 14456, 16384] dup_rows_per_dst=[1958, 0, 140, 6580, 4, 0, 24, 14] combine_rows_per_src=[16376, 16360, 16334, 16380, 9806, 16256, 14456, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130952 payload_rows_remote=114568 dup_rows=120 dup_rows_remote=118 dedup_ratio=0.000916 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 631, 0, 0, 7532, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 58, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 16368, 16384, 16384, 16366, 16384, 16356, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 2, 116] combine_rows_per_src=[16326, 16368, 16384, 16384, 16366, 16384, 16356, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130578 payload_rows_remote=114194 dup_rows=494 dup_rows_remote=494 dedup_ratio=0.003769 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1473, 0, 0, 6706, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 7967, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0] payload_rows_per_dst=[16354, 16384, 16384, 16384, 15934, 16384, 16370, 16384] dup_rows_per_dst=[4, 0, 0, 0, 0, 22, 0, 468] combine_rows_per_src=[16354, 16384, 16384, 16384, 15934, 16384, 16370, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=923910144 dispatch_meta_bytes=458752 combine_payload_bytes=923910144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=924368896 ingress_bytes=999407616 link_bytes=999407616 token_bytes=1848279040 weight_movement_bytes=75497472 total_nvlink_bytes=1923776512 weight_to_token_ratio=0.040847 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.267389534e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125808 payload_rows_remote=109424 dup_rows=5264 dup_rows_remote=5212 dedup_ratio=0.040161 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 266, 0, 0, 8116, 0, 0, 0, 8191, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7972, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15938, 16384, 16380, 16382, 12012, 15944, 16384] dup_rows_per_dst=[380, 10, 374, 0, 0, 0, 4448, 52] combine_rows_per_src=[16384, 15938, 16384, 16380, 16382, 12012, 15944, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=868155392 dispatch_meta_bytes=450952 combine_payload_bytes=868155392 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=868606344 ingress_bytes=1094647808 link_bytes=1094647808 token_bytes=1736761736 weight_movement_bytes=226492416 total_nvlink_bytes=1963254152 weight_to_token_ratio=0.130411 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.388167504e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=938541056 dispatch_meta_bytes=458744 combine_payload_bytes=938541056 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=938999800 ingress_bytes=1014038528 link_bytes=1014038528 token_bytes=1877540856 weight_movement_bytes=75497472 total_nvlink_bytes=1953038328 weight_to_token_ratio=0.040211 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285943590e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=935477248 dispatch_meta_bytes=458752 combine_payload_bytes=935477248 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=935936000 ingress_bytes=1161969664 link_bytes=1161969664 token_bytes=1871413248 weight_movement_bytes=226492416 total_nvlink_bytes=2097905664 weight_to_token_ratio=0.121027 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.473541094e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129252 payload_rows_remote=112868 dup_rows=1820 dup_rows_remote=1732 dedup_ratio=0.013885 router_tokens_per_expert=[0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 10, 0, 0, 7368, 0, 0, 0, 8125, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 19, 0, 27, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 794, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0] payload_rows_per_dst=[14762, 16260, 16330, 16384, 16364, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 48, 0, 88, 78, 1606] combine_rows_per_src=[14762, 16260, 16330, 16384, 16364, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129012 payload_rows_remote=112628 dup_rows=2060 dup_rows_remote=2056 dedup_ratio=0.015717 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 36, 8190, 0, 0, 0, 990, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7438, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7920, 0, 0, 0, 0, 4, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[16384, 16384, 14878, 15848, 16384, 16378, 16372, 16384] dup_rows_per_dst=[0, 2056, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[16384, 16384, 14878, 15848, 16384, 16378, 16372, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=896401408 dispatch_meta_bytes=458544 combine_payload_bytes=896401408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=896859952 ingress_bytes=971898880 link_bytes=971898880 token_bytes=1793261360 weight_movement_bytes=75497472 total_nvlink_bytes=1868758832 weight_to_token_ratio=0.042101 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=63104 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.232504585e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123184 payload_rows_remote=106850 dup_rows=7888 dup_rows_remote=7888 dedup_ratio=0.060181 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 1, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 109, 0, 0, 0, 0, 43, 0, 8129, 0, 1980, 8, 687, 0, 37, 0, 180, 16, 15, 0, 0, 0, 5536, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7843, 0, 0, 0, 0, 33, 3, 8155, 0, 1, 0, 0, 274, 0, 13, 8159, 0, 0, 0, 0, 0, 0, 8, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 7879, 0, 0, 0, 0, 0, 0, 9, 24, 0, 7, 17, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16320, 14044, 12208, 16380, 15740, 15818, 16334, 16340] dup_rows_per_dst=[2, 4854, 2376, 578, 8, 68, 0, 2] combine_rows_per_src=[16320, 14044, 12208, 16380, 15740, 15818, 16334, 16340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=924614656 dispatch_meta_bytes=458400 combine_payload_bytes=924614656 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=925073056 ingress_bytes=1151107072 link_bytes=1151107072 token_bytes=1849687712 weight_movement_bytes=226492416 total_nvlink_bytes=2076180128 weight_to_token_ratio=0.122449 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=62976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.459765798e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130602 payload_rows_remote=114602 dup_rows=470 dup_rows_remote=470 dedup_ratio=0.003586 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 8192, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8000, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 0, 0, 0, 8087, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16346, 16384, 16000, 16340, 16384, 16384, 16382, 16382] dup_rows_per_dst=[0, 452, 0, 0, 10, 0, 8, 0] combine_rows_per_src=[16346, 16384, 16000, 16340, 16384, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=922648576 dispatch_meta_bytes=458736 combine_payload_bytes=922648576 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=923107312 ingress_bytes=1073643520 link_bytes=1073643520 token_bytes=1845755888 weight_movement_bytes=150994944 total_nvlink_bytes=1996750832 weight_to_token_ratio=0.081807 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.361531110e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=875315200 dispatch_meta_bytes=458952 combine_payload_bytes=875315200 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=875774152 ingress_bytes=950812672 link_bytes=950812672 token_bytes=1751089352 weight_movement_bytes=75497472 total_nvlink_bytes=1826586824 weight_to_token_ratio=0.043115 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.205764305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126484 payload_rows_remote=110102 dup_rows=4588 dup_rows_remote=1228 dedup_ratio=0.035004 router_tokens_per_expert=[0, 0, 0, 0, 2, 3, 0, 0, 0, 0, 0, 0, 267, 0, 0, 7435, 2, 0, 0, 8188, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 8189, 1, 0, 2, 0, 0, 8164, 1, 0, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 37, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 539, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 56, 0, 0] payload_rows_per_dst=[16382, 16382, 16242, 16372, 16384, 15838, 16202, 12682] dup_rows_per_dst=[3360, 10, 0, 0, 1132, 0, 56, 30] combine_rows_per_src=[16382, 16382, 16242, 16372, 16384, 15838, 16202, 12682] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130814 payload_rows_remote=114432 dup_rows=258 dup_rows_remote=258 dedup_ratio=0.001968 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8068, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 36, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16384, 16136, 16382, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 256, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[16378, 16384, 16136, 16382, 16382, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128710 payload_rows_remote=112326 dup_rows=2362 dup_rows_remote=2 dedup_ratio=0.018021 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 18, 0, 0, 0, 8042, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 14286, 16120, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2360, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 14286, 16120, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=938819584 dispatch_meta_bytes=460288 combine_payload_bytes=938819584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939279872 ingress_bytes=1014317056 link_bytes=1014317056 token_bytes=1878099456 weight_movement_bytes=75497472 total_nvlink_bytes=1953596928 weight_to_token_ratio=0.040199 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=56576 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286296803e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116938 payload_rows_remote=100554 dup_rows=14134 dup_rows_remote=14086 dedup_ratio=0.107834 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 1702, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6666, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 4, 0, 0, 0, 8023, 0, 0, 0, 0, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 3422, 16102, 15550, 16330, 16382, 16384, 16384] dup_rows_per_dst=[0, 6, 11724, 1032, 812, 508, 4, 48] combine_rows_per_src=[16384, 3422, 16102, 15550, 16330, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=901955584 dispatch_meta_bytes=445320 combine_payload_bytes=901955584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=902400904 ingress_bytes=977453056 link_bytes=977453056 token_bytes=1804356488 weight_movement_bytes=75497472 total_nvlink_bytes=1879853960 weight_to_token_ratio=0.041842 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=58752 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.239548062e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=937426944 dispatch_meta_bytes=458760 combine_payload_bytes=937426944 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=937885704 ingress_bytes=1314914304 link_bytes=1314914304 token_bytes=1875312648 weight_movement_bytes=377487360 total_nvlink_bytes=2252800008 weight_to_token_ratio=0.201293 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.667496426e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129236 payload_rows_remote=112852 dup_rows=1836 dup_rows_remote=1836 dedup_ratio=0.014008 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8084, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 24, 8192, 0, 0, 0, 776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 113, 0, 0, 0, 8102, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 6, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16172, 16384, 14848, 16378, 16344, 16384, 16342, 16384] dup_rows_per_dst=[0, 1690, 0, 52, 0, 0, 94, 0] combine_rows_per_src=[16172, 16384, 14848, 16378, 16344, 16384, 16342, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=920174592 dispatch_meta_bytes=449312 combine_payload_bytes=920174592 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=920623904 ingress_bytes=1297661952 link_bytes=1297661952 token_bytes=1840798496 weight_movement_bytes=377487360 total_nvlink_bytes=2218285856 weight_to_token_ratio=0.205067 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.044118 gemm_active_groups=13 gemm_peak_group_rows=55680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.645618015e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130386 payload_rows_remote=114332 dup_rows=686 dup_rows_remote=686 dedup_ratio=0.005234 router_tokens_per_expert=[0, 0, 0, 0, 196, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 2, 0, 0, 0, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8027, 8191, 0, 5, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 51, 1, 0, 1, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16034, 16378, 16054, 16384, 16384, 16384] dup_rows_per_dst=[390, 0, 0, 232, 0, 56, 0, 8] combine_rows_per_src=[16384, 16384, 16034, 16378, 16054, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130200 payload_rows_remote=113816 dup_rows=872 dup_rows_remote=784 dedup_ratio=0.006653 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 341, 0, 0, 2, 0, 0, 0, 0, 0, 24, 0, 4, 0, 0, 0, 0, 0, 0, 64, 0, 0, 7865, 0, 0, 0, 0, 0, 0, 1, 0, 8122, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 36, 0, 0, 20, 0, 0, 5, 0, 0, 5, 0, 0, 0, 8, 24, 8150, 8097, 0, 0, 12, 0, 12, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 23, 0, 11, 0, 0, 0, 0, 33, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8166, 0, 1, 0, 0, 0, 0, 0, 13, 0, 0] payload_rows_per_dst=[16286, 16380, 16250, 15802, 16364, 16374, 16384, 16360] dup_rows_per_dst=[0, 556, 124, 42, 60, 2, 88, 0] combine_rows_per_src=[16286, 16380, 16250, 15802, 16364, 16374, 16384, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=823738368 dispatch_meta_bytes=458560 combine_payload_bytes=823738368 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=824196928 ingress_bytes=974733312 link_bytes=974733312 token_bytes=1647935296 weight_movement_bytes=150994944 total_nvlink_bytes=1798930240 weight_to_token_ratio=0.091627 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.236099044e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=924483584 dispatch_meta_bytes=458752 combine_payload_bytes=924483584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=924942336 ingress_bytes=999981056 link_bytes=999981056 token_bytes=1849425920 weight_movement_bytes=75497472 total_nvlink_bytes=1924923392 weight_to_token_ratio=0.040822 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.268116737e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130402 payload_rows_remote=114406 dup_rows=670 dup_rows_remote=670 dedup_ratio=0.005112 router_tokens_per_expert=[0, 0, 0, 0, 185, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 7987, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 58, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15996, 16384, 16222, 16328, 16320, 16384] dup_rows_per_dst=[368, 0, 0, 302, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 15996, 16384, 16222, 16328, 16320, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=936607744 dispatch_meta_bytes=460072 combine_payload_bytes=936607744 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937067816 ingress_bytes=1087602688 link_bytes=1087602688 token_bytes=1873675560 weight_movement_bytes=150994944 total_nvlink_bytes=2024670504 weight_to_token_ratio=0.080588 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.379233300e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=932380672 dispatch_meta_bytes=458400 combine_payload_bytes=932380672 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=932839072 ingress_bytes=1007878144 link_bytes=1007878144 token_bytes=1865219744 weight_movement_bytes=75497472 total_nvlink_bytes=1940717216 weight_to_token_ratio=0.040476 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278131356e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126834 payload_rows_remote=110454 dup_rows=4238 dup_rows_remote=3916 dedup_ratio=0.032333 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 4, 0, 0, 1930, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6254, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8139, 0, 7, 0, 1, 1, 1, 8190, 0, 0, 0, 0, 4, 0, 1, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8187, 0, 0, 0, 1, 0, 0, 0, 2, 0, 1, 0, 8191, 0, 0, 0, 2, 0, 50, 0, 0, 0, 3, 0, 3, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16140, 12514, 16384, 16290, 16378, 16368, 16380] dup_rows_per_dst=[322, 3872, 0, 26, 0, 4, 14, 0] combine_rows_per_src=[16380, 16140, 12514, 16384, 16290, 16378, 16368, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130938 payload_rows_remote=114554 dup_rows=134 dup_rows_remote=62 dedup_ratio=0.001022 router_tokens_per_expert=[0, 0, 0, 0, 26, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 28, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16338, 16384, 16344, 16382, 16338, 16384] dup_rows_per_dst=[52, 0, 0, 72, 0, 10, 0, 0] combine_rows_per_src=[16384, 16384, 16338, 16384, 16344, 16382, 16338, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130928 payload_rows_remote=114544 dup_rows=144 dup_rows_remote=144 dedup_ratio=0.001099 router_tokens_per_expert=[0, 0, 0, 0, 45, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8141, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16346, 16384, 16282, 16384, 16380, 16384] dup_rows_per_dst=[90, 0, 0, 24, 0, 30, 0, 0] combine_rows_per_src=[16384, 16384, 16346, 16384, 16282, 16384, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21749736040 prefetch_bytes=1056964608 reduce_bytes=830472192 weight_movement_bytes=1887436800 egress_bytes=10877624936 ingress_bytes=12759547904 link_bytes=12760006600 total_nvlink_bytes=23637172840 weight_to_token_ratio=0.086780 prefetch_share_of_ingress=0.082837 dedup_ratio=0.040305 prefetch_slots=28 unused_prefetch_slots=356 slot_utilization=0.072917 reduce_slots=11 remote_demand=14 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045803 gemm_inner_pad_upper_bound=15367 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.348457040e-04 prefetch_bytes_per_gemm_flop=1.116983252e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20521460920 prefetch_bytes=830472192 reduce_bytes=301989888 weight_movement_bytes=1132462080 egress_bytes=10263422136 ingress_bytes=11390500864 link_bytes=11390500864 total_nvlink_bytes=21653923000 weight_to_token_ratio=0.055184 prefetch_share_of_ingress=0.072909 dedup_ratio=0.086678 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=4 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047181 gemm_inner_pad_upper_bound=14224 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.203729869e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=937213952 dispatch_meta_bytes=460304 combine_payload_bytes=937213952 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937674256 ingress_bytes=1088208896 link_bytes=1088208896 token_bytes=1874888208 weight_movement_bytes=150994944 total_nvlink_bytes=2025883152 weight_to_token_ratio=0.080535 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.380002057e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128592 payload_rows_remote=112208 dup_rows=2480 dup_rows_remote=2476 dedup_ratio=0.018921 router_tokens_per_expert=[0, 0, 0, 0, 81, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 4, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 22, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15062, 15816, 16220, 16376, 16324, 16034, 16376, 16384] dup_rows_per_dst=[162, 0, 552, 32, 1258, 44, 428, 4] combine_rows_per_src=[15062, 15816, 16220, 16376, 16324, 16034, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=904839168 dispatch_meta_bytes=457480 combine_payload_bytes=904839168 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=905296648 ingress_bytes=980336640 link_bytes=980336640 token_bytes=1810135816 weight_movement_bytes=75497472 total_nvlink_bytes=1885633288 weight_to_token_ratio=0.041708 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.243204852e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938426368 dispatch_meta_bytes=458464 combine_payload_bytes=938426368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938884832 ingress_bytes=1013923840 link_bytes=1013923840 token_bytes=1877311200 weight_movement_bytes=75497472 total_nvlink_bytes=1952808672 weight_to_token_ratio=0.040216 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285798150e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130304 payload_rows_remote=113920 dup_rows=768 dup_rows_remote=624 dedup_ratio=0.005859 router_tokens_per_expert=[0, 0, 0, 0, 170, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 39, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 109, 0, 0, 0, 0, 0, 0, 8067, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16226, 16384, 16024, 16384, 16134, 16384] dup_rows_per_dst=[326, 0, 0, 280, 0, 144, 0, 18] combine_rows_per_src=[16384, 16384, 16226, 16384, 16024, 16384, 16134, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938344448 dispatch_meta_bytes=458752 combine_payload_bytes=938344448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938803200 ingress_bytes=1013841920 link_bytes=1013841920 token_bytes=1877147648 weight_movement_bytes=75497472 total_nvlink_bytes=1952645120 weight_to_token_ratio=0.040219 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285694264e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21631779312 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=10818634224 ingress_bytes=12247597056 link_bytes=12247597056 total_nvlink_bytes=23066231280 weight_to_token_ratio=0.066312 prefetch_share_of_ingress=0.061643 dedup_ratio=0.038165 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=9 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045190 gemm_inner_pad_upper_bound=16002 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.294306421e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=919207936 dispatch_meta_bytes=458736 combine_payload_bytes=919207936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919666672 ingress_bytes=994705408 link_bytes=994705408 token_bytes=1838874608 weight_movement_bytes=75497472 total_nvlink_bytes=1914372080 weight_to_token_ratio=0.041056 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.261426473e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21657713440 prefetch_bytes=679477248 reduce_bytes=679477248 weight_movement_bytes=1358954496 egress_bytes=10831608608 ingress_bytes=12185059328 link_bytes=12185059328 total_nvlink_bytes=23016667936 weight_to_token_ratio=0.062747 prefetch_share_of_ingress=0.055763 dedup_ratio=0.039860 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=9 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044424 gemm_inner_pad_upper_bound=16256 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.287697534e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21173851656 prefetch_bytes=830472192 reduce_bytes=1887436800 weight_movement_bytes=2717908992 egress_bytes=10589672968 ingress_bytes=13302087680 link_bytes=13302087680 total_nvlink_bytes=23891760648 weight_to_token_ratio=0.128362 prefetch_share_of_ingress=0.062432 dedup_ratio=0.056455 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=25 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045496 gemm_inner_pad_upper_bound=16129 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.405743299e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=933232640 dispatch_meta_bytes=458176 combine_payload_bytes=933232640 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=933690816 ingress_bytes=1084227584 link_bytes=1084227584 token_bytes=1866923456 weight_movement_bytes=150994944 total_nvlink_bytes=2017918400 weight_to_token_ratio=0.080879 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.374953193e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20175207696 prefetch_bytes=905969664 reduce_bytes=830472192 weight_movement_bytes=1736441856 egress_bytes=10090298640 ingress_bytes=11821350912 link_bytes=11821350912 total_nvlink_bytes=21911649552 weight_to_token_ratio=0.086068 prefetch_share_of_ingress=0.076638 dedup_ratio=0.097074 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=11 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044424 gemm_inner_pad_upper_bound=16129 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.249261411e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130254 payload_rows_remote=113872 dup_rows=818 dup_rows_remote=784 dedup_ratio=0.006241 router_tokens_per_expert=[0, 0, 0, 0, 17, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 5, 0, 2, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 8164, 1, 0, 0, 0, 0, 2, 0, 0, 0, 8, 0, 0, 0, 0, 125, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 1, 2, 0, 0, 0, 1, 8063, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 72, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 15978, 16130, 16378, 16240, 16378] dup_rows_per_dst=[34, 16, 602, 18, 10, 138, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 15978, 16130, 16378, 16240, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21455491088 prefetch_bytes=1132462080 reduce_bytes=905969664 weight_movement_bytes=2038431744 egress_bytes=10730491920 ingress_bytes=12763430912 link_bytes=12763430912 total_nvlink_bytes=23493922832 weight_to_token_ratio=0.095007 prefetch_share_of_ingress=0.088727 dedup_ratio=0.045479 prefetch_slots=30 unused_prefetch_slots=354 slot_utilization=0.078125 reduce_slots=12 remote_demand=15 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044730 gemm_inner_pad_upper_bound=16129 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.348818915e-04 prefetch_bytes_per_gemm_flop=1.196767770e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=932839424 dispatch_meta_bytes=458624 combine_payload_bytes=932839424 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=933298048 ingress_bytes=1234829312 link_bytes=1234829312 token_bytes=1866137472 weight_movement_bytes=301989888 total_nvlink_bytes=2168127360 weight_to_token_ratio=0.161826 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.565937383e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20381307576 prefetch_bytes=754974720 reduce_bytes=830472192 weight_movement_bytes=1585446912 egress_bytes=10193375928 ingress_bytes=11773378560 link_bytes=11773378560 total_nvlink_bytes=21966754488 weight_to_token_ratio=0.077789 prefetch_share_of_ingress=0.064126 dedup_ratio=0.084904 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=11 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.046875 gemm_inner_pad_upper_bound=14351 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.244191770e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none + [2026-08-29 23:35:11] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 654.2 | throughput per GPU (TFLOP/s/GPU): 276.4 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.049876E+01 | loss scale: 1.0 | grad norm: 95.167 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105016 payload_rows_remote=91138 dup_rows=26056 dup_rows_remote=20824 dedup_ratio=0.198792 router_tokens_per_expert=[0, 0, 0, 1, 211, 70, 6275, 0, 0, 7432, 0, 0, 0, 0, 191, 1, 0, 77, 0, 0, 438, 0, 0, 3099, 4551, 0, 0, 8178, 189, 910, 0, 1464, 0, 0, 422, 0, 1, 55, 0, 0, 3, 1, 2310, 248, 0, 5661, 0, 854, 0, 0, 269, 0, 0, 0, 0, 0, 108, 0, 36, 534, 0, 0, 0, 0, 2315, 1, 0, 0, 627, 0, 1, 0, 772, 0, 809, 890, 0, 0, 36, 0, 0, 0, 0, 0, 114, 0, 0, 1614, 0, 243, 0, 0, 1, 1, 0, 0, 0, 1876, 0, 0, 1, 1, 0, 0, 16, 1323, 326, 6, 0, 84, 3289, 0, 0, 5488, 0, 0, 0, 2, 1645, 0, 0, 0, 12, 452, 0, 0, 2, 0] payload_rows_per_dst=[12726, 10288, 13878, 16364, 12834, 15268, 10956, 12702] dup_rows_per_dst=[772, 3770, 5232, 1886, 5466, 3542, 2888, 2500] combine_rows_per_src=[12726, 10288, 13878, 16364, 12834, 15268, 10956, 12702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105722 payload_rows_remote=89366 dup_rows=25350 dup_rows_remote=23306 dedup_ratio=0.193405 router_tokens_per_expert=[0, 0, 0, 1, 272, 137, 6397, 0, 0, 6862, 0, 3, 2, 0, 196, 1, 0, 28, 0, 23, 422, 0, 0, 3381, 5610, 1, 0, 8174, 275, 1054, 2, 2020, 3, 4, 344, 0, 1, 7, 0, 2, 0, 1, 2123, 105, 0, 4854, 3, 1214, 0, 0, 60, 0, 4, 0, 0, 0, 38, 0, 29, 891, 0, 0, 0, 4, 1232, 1, 0, 0, 440, 0, 0, 0, 748, 0, 1260, 928, 12, 1, 16, 0, 0, 0, 0, 5, 102, 0, 0, 1346, 0, 136, 0, 2, 0, 0, 2, 0, 0, 1055, 3, 0, 2, 0, 0, 0, 15, 1489, 436, 0, 1, 10, 3072, 0, 0, 6725, 0, 0, 1, 0, 1345, 0, 0, 0, 19, 576, 4, 0, 4, 0] payload_rows_per_dst=[13042, 10578, 12744, 16356, 13970, 14398, 10230, 14404] dup_rows_per_dst=[976, 3834, 4578, 2044, 6526, 2512, 1936, 2944] combine_rows_per_src=[13042, 10578, 12744, 16356, 13970, 14398, 10230, 14404] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105538 payload_rows_remote=90748 dup_rows=25534 dup_rows_remote=22182 dedup_ratio=0.194809 router_tokens_per_expert=[0, 0, 2, 0, 371, 74, 6700, 0, 0, 7109, 0, 0, 2, 1, 166, 0, 0, 37, 0, 0, 473, 0, 0, 3880, 5168, 0, 0, 8136, 371, 773, 1, 1362, 0, 0, 599, 0, 0, 4, 0, 0, 2, 0, 2194, 75, 0, 5794, 0, 1497, 0, 5, 338, 0, 0, 0, 0, 0, 43, 0, 13, 908, 0, 2, 0, 1, 1526, 0, 0, 0, 477, 2, 0, 1, 775, 0, 826, 347, 1, 4, 8, 0, 1, 2, 0, 0, 66, 0, 0, 1169, 1, 139, 0, 9, 0, 0, 0, 1, 0, 944, 0, 0, 0, 0, 0, 2, 6, 946, 192, 0, 0, 14, 2905, 0, 0, 6951, 1, 0, 0, 6, 1502, 2, 0, 2, 16, 591, 0, 0, 0, 0] payload_rows_per_dst=[13660, 6224, 11870, 16308, 13394, 14654, 14638, 14790] dup_rows_per_dst=[972, 882, 3560, 2584, 4876, 2340, 6968, 3352] combine_rows_per_src=[13660, 6224, 11870, 16308, 13394, 14654, 14638, 14790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104918 payload_rows_remote=91216 dup_rows=26154 dup_rows_remote=20494 dedup_ratio=0.199539 router_tokens_per_expert=[0, 0, 0, 0, 176, 91, 6083, 0, 0, 6716, 4, 0, 0, 0, 313, 0, 2, 63, 0, 5, 460, 0, 1, 3191, 5220, 0, 0, 8190, 199, 685, 1, 1524, 0, 0, 213, 0, 1, 1, 0, 2, 4, 0, 2239, 106, 0, 5309, 1, 1968, 0, 7, 179, 0, 0, 0, 0, 0, 102, 4, 28, 668, 0, 0, 0, 4, 1447, 0, 0, 0, 460, 0, 0, 1, 701, 0, 1402, 421, 6, 5, 18, 0, 0, 2, 2, 0, 168, 0, 0, 1786, 1, 162, 0, 3, 4, 1, 3, 0, 0, 560, 1, 0, 0, 1, 0, 0, 22, 2086, 569, 0, 0, 27, 2958, 0, 0, 6579, 0, 0, 0, 0, 1691, 0, 0, 3, 61, 623, 0, 0, 2, 0] payload_rows_per_dst=[12362, 9454, 13862, 16380, 13702, 14418, 10242, 14498] dup_rows_per_dst=[972, 2808, 5826, 1984, 5660, 3278, 2206, 3420] combine_rows_per_src=[12362, 9454, 13862, 16380, 13702, 14418, 10242, 14498] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=746602496 dispatch_meta_bytes=447848 combine_payload_bytes=746602496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=747050344 ingress_bytes=897597440 link_bytes=897597440 token_bytes=1493652840 weight_movement_bytes=150994944 total_nvlink_bytes=1644647784 weight_to_token_ratio=0.101091 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.034926 gemm_active_groups=17 gemm_peak_group_rows=27008 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.138279900e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=732086272 dispatch_meta_bytes=450688 combine_payload_bytes=732086272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=732536960 ingress_bytes=807583744 link_bytes=807583744 token_bytes=1464623232 weight_movement_bytes=75497472 total_nvlink_bytes=1540120704 weight_to_token_ratio=0.051547 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.038603 gemm_active_groups=15 gemm_peak_group_rows=57088 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.024129863e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104424 payload_rows_remote=91404 dup_rows=26648 dup_rows_remote=20234 dedup_ratio=0.203308 router_tokens_per_expert=[1, 0, 0, 3, 246, 118, 6363, 2, 0, 7754, 0, 0, 0, 1, 27, 0, 0, 28, 0, 2, 442, 0, 1, 3249, 4499, 0, 0, 8154, 178, 1381, 0, 1821, 0, 0, 670, 0, 0, 5, 0, 0, 0, 0, 2985, 91, 1, 5545, 0, 1129, 0, 0, 79, 1, 0, 0, 0, 2, 89, 0, 20, 726, 0, 0, 0, 4, 1609, 2, 0, 0, 240, 0, 3, 0, 746, 0, 739, 410, 4, 1, 12, 0, 0, 0, 0, 1, 16, 0, 0, 1558, 1, 160, 0, 3, 19, 0, 0, 2, 0, 648, 2, 0, 0, 2, 2, 1, 3, 937, 200, 0, 0, 15, 4108, 1, 0, 6352, 0, 0, 0, 0, 1459, 0, 0, 2, 13, 648, 0, 0, 0, 0] payload_rows_per_dst=[14198, 13020, 14370, 16088, 6642, 15714, 10304, 14088] dup_rows_per_dst=[3388, 6414, 6482, 1766, 890, 3314, 1534, 2860] combine_rows_per_src=[14198, 13020, 14370, 16088, 6642, 15714, 10304, 14088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=743407616 dispatch_meta_bytes=451720 combine_payload_bytes=743407616 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=743859336 ingress_bytes=818905088 link_bytes=818905088 token_bytes=1487266952 weight_movement_bytes=75497472 total_nvlink_bytes=1562764424 weight_to_token_ratio=0.050763 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.034926 gemm_active_groups=16 gemm_peak_group_rows=48000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.038486921e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104410 payload_rows_remote=89494 dup_rows=26662 dup_rows_remote=24912 dedup_ratio=0.203415 router_tokens_per_expert=[0, 0, 0, 1, 511, 80, 5656, 0, 0, 7306, 0, 1, 1, 0, 59, 0, 1, 36, 0, 1, 535, 0, 1, 4133, 4758, 1, 0, 8143, 230, 965, 1, 1510, 0, 0, 535, 0, 0, 8, 0, 0, 20, 0, 2792, 52, 0, 5167, 1, 1228, 0, 1, 117, 2, 10, 0, 0, 0, 56, 1, 17, 973, 0, 1, 0, 0, 1779, 0, 0, 0, 201, 0, 4, 0, 867, 0, 880, 455, 0, 5, 21, 0, 0, 0, 0, 0, 48, 0, 0, 887, 2, 72, 0, 13, 4, 1, 0, 0, 0, 1088, 2, 0, 1, 1, 1, 0, 11, 1713, 315, 1, 0, 54, 4227, 2, 0, 5818, 0, 1, 0, 3, 1592, 0, 0, 0, 7, 547, 0, 0, 0, 2] payload_rows_per_dst=[11708, 10758, 10126, 16348, 13180, 14916, 14090, 13284] dup_rows_per_dst=[910, 4070, 2604, 2294, 4760, 1750, 7618, 2656] combine_rows_per_src=[11708, 10758, 10126, 16348, 13180, 14916, 14090, 13284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=747241472 dispatch_meta_bytes=446840 combine_payload_bytes=747241472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=747688312 ingress_bytes=822738944 link_bytes=822738944 token_bytes=1494929784 weight_movement_bytes=75497472 total_nvlink_bytes=1570427256 weight_to_token_ratio=0.050502 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.038603 gemm_active_groups=15 gemm_peak_group_rows=29440 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.043348790e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104820 payload_rows_remote=90010 dup_rows=26252 dup_rows_remote=17948 dedup_ratio=0.200287 router_tokens_per_expert=[0, 0, 0, 0, 150, 167, 6489, 0, 0, 7392, 7, 0, 0, 0, 166, 1, 0, 62, 0, 7, 513, 0, 1, 3811, 4959, 0, 0, 8173, 275, 1084, 0, 1095, 10, 4, 211, 0, 2, 18, 1, 0, 0, 1, 2675, 161, 4, 5049, 9, 1606, 0, 0, 131, 0, 0, 2, 0, 0, 85, 1, 16, 710, 2, 0, 0, 1, 1649, 4, 0, 1, 598, 0, 0, 0, 616, 0, 999, 1021, 1, 0, 23, 0, 0, 0, 0, 1, 91, 0, 0, 855, 0, 208, 4, 0, 5, 0, 0, 0, 0, 873, 0, 1, 1, 6, 0, 0, 31, 2235, 546, 15, 0, 29, 2771, 0, 0, 5663, 0, 3, 0, 9, 1568, 0, 1, 1, 38, 611, 1, 2, 4, 0] payload_rows_per_dst=[13142, 10436, 8406, 16360, 13636, 15036, 14810, 12994] dup_rows_per_dst=[818, 3260, 998, 1882, 6106, 2076, 8304, 2808] combine_rows_per_src=[13142, 10436, 8406, 16360, 13636, 15036, 14810, 12994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=83926 payload_rows_remote=67934 dup_rows=47146 dup_rows_remote=31712 dedup_ratio=0.359695 router_tokens_per_expert=[0, 0, 0, 1, 403, 52, 5755, 0, 0, 7783, 0, 0, 0, 0, 158, 0, 0, 70, 0, 15, 487, 0, 7, 3774, 4912, 1, 0, 8174, 284, 1789, 0, 1250, 0, 18, 585, 0, 4, 38, 0, 2, 8, 0, 2296, 193, 0, 3818, 0, 1306, 0, 0, 443, 0, 0, 0, 0, 0, 115, 0, 35, 619, 0, 0, 0, 2, 1786, 1, 0, 0, 561, 0, 1, 1, 842, 0, 896, 601, 1, 12, 13, 0, 0, 1, 1, 1, 139, 0, 0, 1299, 3, 264, 0, 2, 5, 0, 0, 0, 0, 1004, 0, 0, 0, 6, 0, 0, 3, 2329, 373, 0, 0, 26, 3433, 0, 0, 4981, 0, 0, 0, 2, 1959, 0, 0, 0, 27, 563, 0, 0, 3, 0] payload_rows_per_dst=[15992, 16384, 12052, 2152, 7808, 6932, 11266, 11340] dup_rows_per_dst=[15434, 18892, 4484, 276, 1622, 924, 3082, 2432] combine_rows_per_src=[15992, 16384, 12052, 2152, 7808, 6932, 11266, 11340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130554 payload_rows_remote=114172 dup_rows=518 dup_rows_remote=406 dedup_ratio=0.003952 router_tokens_per_expert=[8192, 0, 8008, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 8179, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8124, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0] payload_rows_per_dst=[16016, 16358, 16382, 16264, 16384, 16384, 16382, 16384] dup_rows_per_dst=[4, 0, 112, 306, 0, 18, 0, 78] combine_rows_per_src=[16016, 16358, 16382, 16264, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=748781568 dispatch_meta_bytes=446552 combine_payload_bytes=748781568 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=749228120 ingress_bytes=1126268928 link_bytes=1126268928 token_bytes=1498009688 weight_movement_bytes=377487360 total_nvlink_bytes=1875497048 weight_to_token_ratio=0.251993 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.038603 gemm_active_groups=15 gemm_peak_group_rows=19456 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.428267535e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=733134848 dispatch_meta_bytes=457624 combine_payload_bytes=733134848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=733592472 ingress_bytes=808632320 link_bytes=808632320 token_bytes=1466727320 weight_movement_bytes=75497472 total_nvlink_bytes=1542224792 weight_to_token_ratio=0.051473 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=52864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.025459605e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130718 payload_rows_remote=114504 dup_rows=354 dup_rows_remote=222 dedup_ratio=0.002701 router_tokens_per_expert=[8192, 0, 8105, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7911, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 261, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16216, 16370, 16382, 16214, 16384, 16384, 16384, 16384] dup_rows_per_dst=[6, 0, 130, 132, 0, 2, 0, 84] combine_rows_per_src=[16216, 16370, 16382, 16214, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=935297024 dispatch_meta_bytes=458312 combine_payload_bytes=935297024 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=935755336 ingress_bytes=1086291968 link_bytes=1086291968 token_bytes=1871052360 weight_movement_bytes=150994944 total_nvlink_bytes=2022047304 weight_to_token_ratio=0.080701 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=63744 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.377571123e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130050 payload_rows_remote=113666 dup_rows=1022 dup_rows_remote=1022 dedup_ratio=0.007797 router_tokens_per_expert=[8192, 0, 7705, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15414, 16380, 16382, 16338, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 334, 588, 0, 14, 0, 86] combine_rows_per_src=[15414, 16380, 16382, 16338, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126374 payload_rows_remote=109990 dup_rows=4698 dup_rows_remote=4392 dedup_ratio=0.035843 router_tokens_per_expert=[8192, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8192, 8149, 0, 8185, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8184, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8089, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 2, 150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[14796, 16310, 13548, 16200, 16384, 16372, 16380, 16384] dup_rows_per_dst=[0, 382, 58, 3938, 2, 12, 0, 306] combine_rows_per_src=[14796, 16310, 13548, 16200, 16384, 16372, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=556515328 dispatch_meta_bytes=398584 combine_payload_bytes=556515328 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=556913912 ingress_bytes=707510272 link_bytes=707510272 token_bytes=1113429240 weight_movement_bytes=150994944 total_nvlink_bytes=1264424184 weight_to_token_ratio=0.135613 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=49792 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.972226145e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=737361920 dispatch_meta_bytes=431832 combine_payload_bytes=737361920 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=737793752 ingress_bytes=812859392 link_bytes=812859392 token_bytes=1475155672 weight_movement_bytes=75497472 total_nvlink_bytes=1550653144 weight_to_token_ratio=0.051179 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.034926 gemm_active_groups=17 gemm_peak_group_rows=26880 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.030820128e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=938016768 dispatch_meta_bytes=458904 combine_payload_bytes=938016768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938475672 ingress_bytes=1013514240 link_bytes=1013514240 token_bytes=1876492440 weight_movement_bytes=75497472 total_nvlink_bytes=1951989912 weight_to_token_ratio=0.040233 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=57728 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285278719e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=931151872 dispatch_meta_bytes=458752 combine_payload_bytes=931151872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=931610624 ingress_bytes=1006649344 link_bytes=1006649344 token_bytes=1862762496 weight_movement_bytes=75497472 total_nvlink_bytes=1938259968 weight_to_token_ratio=0.040530 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.276573065e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=901038080 dispatch_meta_bytes=457528 combine_payload_bytes=901038080 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=901495608 ingress_bytes=976535552 link_bytes=976535552 token_bytes=1802533688 weight_movement_bytes=75497472 total_nvlink_bytes=1878031160 weight_to_token_ratio=0.041884 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=62336 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.238384538e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125350 payload_rows_remote=109310 dup_rows=5722 dup_rows_remote=5722 dedup_ratio=0.043655 router_tokens_per_expert=[8189, 0, 7912, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8017, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 8187, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3135, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 2784, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 8191, 0, 2105, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 2, 0, 1, 0] payload_rows_per_dst=[15824, 16040, 16380, 11578, 16380, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 1218, 266, 2, 4, 8, 4224] combine_rows_per_src=[15824, 16040, 16380, 11578, 16380, 16382, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129278 payload_rows_remote=112896 dup_rows=1794 dup_rows_remote=1716 dedup_ratio=0.013687 router_tokens_per_expert=[8190, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8189, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 264, 0, 0, 8179, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6953, 5, 0, 0, 0, 0, 0, 3, 0, 0, 0, 453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 573, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[16216, 16380, 16380, 14772, 16382, 16382, 16384, 16382] dup_rows_per_dst=[0, 0, 508, 56, 0, 78, 0, 1152] combine_rows_per_src=[16216, 16380, 16380, 14772, 16382, 16382, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116280 payload_rows_remote=99896 dup_rows=14792 dup_rows_remote=268 dedup_ratio=0.112854 router_tokens_per_expert=[0, 0, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7252, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 8192, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 798, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 1596] dup_rows_per_dst=[232, 0, 14524, 36, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 1596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=895467520 dispatch_meta_bytes=460128 combine_payload_bytes=895467520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=895927648 ingress_bytes=1121959936 link_bytes=1121959936 token_bytes=1791395168 weight_movement_bytes=226492416 total_nvlink_bytes=2017887584 weight_to_token_ratio=0.126434 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.422803127e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124024 payload_rows_remote=108058 dup_rows=7048 dup_rows_remote=2450 dedup_ratio=0.053772 router_tokens_per_expert=[8192, 0, 7049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8188, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15966, 16384, 16384, 16332, 16374, 16280, 16380, 9924] dup_rows_per_dst=[4598, 106, 202, 2082, 0, 0, 0, 60] combine_rows_per_src=[15966, 16384, 16384, 16332, 16374, 16280, 16380, 9924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129498 payload_rows_remote=113114 dup_rows=1574 dup_rows_remote=1470 dedup_ratio=0.012009 router_tokens_per_expert=[0, 0, 87, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 52, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 7424, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16346, 16384, 16384, 16384, 16384, 14848] dup_rows_per_dst=[174, 0, 1252, 104, 0, 24, 20, 0] combine_rows_per_src=[16384, 16384, 16346, 16384, 16384, 16384, 16384, 14848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=924844032 dispatch_meta_bytes=458448 combine_payload_bytes=924844032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=925302480 ingress_bytes=1000341504 link_bytes=1000341504 token_bytes=1850146512 weight_movement_bytes=75497472 total_nvlink_bytes=1925643984 weight_to_token_ratio=0.040806 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.268573836e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130136 payload_rows_remote=113754 dup_rows=936 dup_rows_remote=932 dedup_ratio=0.007141 router_tokens_per_expert=[8185, 0, 7992, 4, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 5, 0, 2, 0, 2, 8186, 8174, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 150, 0, 0, 8185, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7204, 0, 5, 3, 0, 0, 1, 0, 2, 0, 0, 787, 1, 0, 0, 0, 1, 4, 0, 4, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 238, 1, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16010, 16360, 16374, 15870, 16384, 16374, 16382, 16382] dup_rows_per_dst=[6, 2, 298, 136, 12, 2, 4, 476] combine_rows_per_src=[16010, 16360, 16374, 15870, 16384, 16374, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=818348032 dispatch_meta_bytes=400656 combine_payload_bytes=818348032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=818748688 ingress_bytes=893845504 link_bytes=893845504 token_bytes=1637096720 weight_movement_bytes=75497472 total_nvlink_bytes=1712594192 weight_to_token_ratio=0.046117 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=34432 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.133521917e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121966 payload_rows_remote=105582 dup_rows=9106 dup_rows_remote=9106 dedup_ratio=0.069473 router_tokens_per_expert=[0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 4342, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 3808, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 9642, 16384, 16384, 16382, 16384, 14022] dup_rows_per_dst=[66, 0, 2290, 20, 0, 0, 6, 6724] combine_rows_per_src=[16384, 16384, 9642, 16384, 16384, 16382, 16384, 14022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127796 payload_rows_remote=111412 dup_rows=3276 dup_rows_remote=148 dedup_ratio=0.024994 router_tokens_per_expert=[0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 7058, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 8186, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 6, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 1119, 0, 8183, 0, 0] payload_rows_per_dst=[15806, 16006, 14164, 16312, 16384, 16384, 16356, 16384] dup_rows_per_dst=[32, 0, 0, 20, 80, 16, 0, 3128] combine_rows_per_src=[15806, 16006, 14164, 16312, 16384, 16384, 16356, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=885211136 dispatch_meta_bytes=442032 combine_payload_bytes=885211136 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=885653168 ingress_bytes=1187201024 link_bytes=1187201024 token_bytes=1770864304 weight_movement_bytes=301989888 total_nvlink_bytes=2072854192 weight_to_token_ratio=0.170532 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=62336 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.505538010e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=931872768 dispatch_meta_bytes=458744 combine_payload_bytes=931872768 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=932331512 ingress_bytes=1007370240 link_bytes=1007370240 token_bytes=1864204280 weight_movement_bytes=75497472 total_nvlink_bytes=1939701752 weight_to_token_ratio=0.040498 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.277487262e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=926629888 dispatch_meta_bytes=458336 combine_payload_bytes=926629888 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=927088224 ingress_bytes=1228619776 link_bytes=1228619776 token_bytes=1853718112 weight_movement_bytes=301989888 total_nvlink_bytes=2155708000 weight_to_token_ratio=0.162910 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.558062817e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=864927744 dispatch_meta_bytes=458752 combine_payload_bytes=864927744 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=865386496 ingress_bytes=1015922688 link_bytes=1015922688 token_bytes=1730314240 weight_movement_bytes=150994944 total_nvlink_bytes=1881309184 weight_to_token_ratio=0.087264 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.288332970e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130680 payload_rows_remote=114312 dup_rows=392 dup_rows_remote=392 dedup_ratio=0.002991 router_tokens_per_expert=[0, 3, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 1, 120, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 8179, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 4, 2, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 2, 3, 0, 2, 0, 8014, 0, 8188, 0, 0] payload_rows_per_dst=[16384, 16368, 16374, 16380, 16376, 16382, 16382, 16034] dup_rows_per_dst=[10, 0, 252, 104, 10, 8, 0, 8] combine_rows_per_src=[16384, 16368, 16374, 16380, 16376, 16382, 16382, 16034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=912687104 dispatch_meta_bytes=446240 combine_payload_bytes=912687104 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=913133344 ingress_bytes=1139179520 link_bytes=1139179520 token_bytes=1825820448 weight_movement_bytes=226492416 total_nvlink_bytes=2052312864 weight_to_token_ratio=0.124050 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=34048 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.444639983e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116902 payload_rows_remote=100518 dup_rows=14170 dup_rows_remote=14134 dedup_ratio=0.108109 router_tokens_per_expert=[0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1089, 0, 0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 2260, 16384, 16378, 16384, 16384, 16344] dup_rows_per_dst=[36, 0, 0, 6, 0, 36, 0, 14092] combine_rows_per_src=[16384, 16384, 2260, 16384, 16378, 16384, 16384, 16344] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123204 payload_rows_remote=106820 dup_rows=7868 dup_rows_remote=7862 dedup_ratio=0.060028 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 166, 0, 0, 0, 0, 3764, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7994, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4454, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 9, 0, 0, 0, 0, 8188, 8184, 0, 8192, 0] payload_rows_per_dst=[16376, 16384, 16384, 15992, 8918, 16382, 16384, 16384] dup_rows_per_dst=[0, 7854, 6, 0, 4, 2, 0, 2] combine_rows_per_src=[16376, 16384, 16384, 15992, 8918, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=936443904 dispatch_meta_bytes=458816 combine_payload_bytes=936443904 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=936902720 ingress_bytes=1162936320 link_bytes=1162936320 token_bytes=1873346624 weight_movement_bytes=226492416 total_nvlink_bytes=2099839040 weight_to_token_ratio=0.120903 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.474766949e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=823443456 dispatch_meta_bytes=458608 combine_payload_bytes=823443456 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=823902064 ingress_bytes=898940928 link_bytes=898940928 token_bytes=1647345520 weight_movement_bytes=75497472 total_nvlink_bytes=1722842992 weight_to_token_ratio=0.045830 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.139983632e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125848 payload_rows_remote=109464 dup_rows=5224 dup_rows_remote=6 dedup_ratio=0.039856 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 6493, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7247, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 2606, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 8178, 0] payload_rows_per_dst=[16384, 13048, 16356, 14524, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 6, 0, 0, 5218, 0, 0, 0] combine_rows_per_src=[16384, 13048, 16356, 14524, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121328 payload_rows_remote=114158 dup_rows=9744 dup_rows_remote=9744 dedup_ratio=0.074341 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 8158, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3560, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 4258, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8169, 8192, 0, 8189, 0] payload_rows_per_dst=[16338, 16320, 16378, 7170, 15970, 16384, 16384, 16384] dup_rows_per_dst=[0, 4990, 0, 0, 4754, 0, 0, 0] combine_rows_per_src=[16338, 16320, 16378, 7170, 15970, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114788 payload_rows_remote=98404 dup_rows=16284 dup_rows_remote=16270 dedup_ratio=0.124237 router_tokens_per_expert=[0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7876, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 263, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16072, 16384, 16270, 16384, 526] dup_rows_per_dst=[14, 436, 15750, 72, 0, 0, 12, 0] combine_rows_per_src=[16384, 16384, 16384, 16072, 16384, 16270, 16384, 526] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=875069440 dispatch_meta_bytes=458728 combine_payload_bytes=875069440 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=875528168 ingress_bytes=1026064384 link_bytes=1026064384 token_bytes=1750597608 weight_movement_bytes=150994944 total_nvlink_bytes=1901592552 weight_to_token_ratio=0.086253 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.301194068e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119988 payload_rows_remote=103616 dup_rows=11084 dup_rows_remote=11080 dedup_ratio=0.084564 router_tokens_per_expert=[0, 1, 14, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 8178, 0, 6, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 2721, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 7, 8142, 0, 18, 0, 0, 0, 0, 0, 6, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 7, 0, 0, 0, 8184, 0, 8, 0, 0, 4, 8, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 2, 4, 7, 0, 0, 3, 0, 5515, 0, 8180, 0, 0] payload_rows_per_dst=[16382, 16342, 5442, 16384, 16302, 16384, 16372, 16380] dup_rows_per_dst=[16, 0, 8, 26, 6, 38, 4, 10986] combine_rows_per_src=[16382, 16342, 5442, 16384, 16302, 16384, 16372, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114978 payload_rows_remote=98682 dup_rows=16094 dup_rows_remote=15232 dedup_ratio=0.122787 router_tokens_per_expert=[6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 366, 0, 0, 0, 0, 6394, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5888, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3724, 0, 0, 0, 0, 0, 0, 12, 60, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 22, 0, 0, 0, 0, 8174, 8126, 0, 8192, 0] payload_rows_per_dst=[16360, 732, 16384, 16126, 16384, 16312, 16384, 16296] dup_rows_per_dst=[0, 0, 2, 7648, 7550, 32, 0, 862] combine_rows_per_src=[16360, 732, 16384, 16126, 16384, 16312, 16384, 16296] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=896729088 dispatch_meta_bytes=437880 combine_payload_bytes=896729088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=897166968 ingress_bytes=972226560 link_bytes=972226560 token_bytes=1793896056 weight_movement_bytes=75497472 total_nvlink_bytes=1869393528 weight_to_token_ratio=0.042086 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=38912 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.232920129e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=935182336 dispatch_meta_bytes=495608 combine_payload_bytes=935182336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935677944 ingress_bytes=1010679808 link_bytes=1010679808 token_bytes=1870860280 weight_movement_bytes=75497472 total_nvlink_bytes=1946357752 weight_to_token_ratio=0.040354 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=50048 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281684260e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=806125568 dispatch_meta_bytes=458696 combine_payload_bytes=806125568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=806584264 ingress_bytes=881623040 link_bytes=881623040 token_bytes=1612709832 weight_movement_bytes=75497472 total_nvlink_bytes=1688207304 weight_to_token_ratio=0.046814 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.118022113e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 2, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=848822272 dispatch_meta_bytes=458784 combine_payload_bytes=848822272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=849281056 ingress_bytes=924319744 link_bytes=924319744 token_bytes=1698103328 weight_movement_bytes=75497472 total_nvlink_bytes=1773600800 weight_to_token_ratio=0.044460 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.172167543e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=808402944 dispatch_meta_bytes=455656 combine_payload_bytes=808402944 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=808858600 ingress_bytes=1261387776 link_bytes=1261387776 token_bytes=1617261544 weight_movement_bytes=452984832 total_nvlink_bytes=2070246376 weight_to_token_ratio=0.280094 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.599617254e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120310 payload_rows_remote=103926 dup_rows=10762 dup_rows_remote=10746 dedup_ratio=0.082108 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 460, 0, 0, 0, 0, 7277, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5080, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3548, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 8184, 8177, 0, 8192, 0] payload_rows_per_dst=[16374, 10640, 16384, 11402, 16384, 16384, 16384, 16358] dup_rows_per_dst=[0, 500, 0, 3094, 7150, 16, 0, 2] combine_rows_per_src=[16374, 10640, 16384, 11402, 16384, 16384, 16384, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111984 payload_rows_remote=95600 dup_rows=19088 dup_rows_remote=64 dedup_ratio=0.145630 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 1, 0, 0, 0, 11, 0, 0, 0, 1535, 0, 11, 0, 0, 8008, 0, 8146, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6433, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 550, 0, 0, 0, 3, 0, 0, 9, 46, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 8175, 0, 0, 0, 0, 2, 8176, 8006, 0, 8189, 3] payload_rows_per_dst=[16376, 16384, 16384, 12872, 1198, 16364, 16380, 16026] dup_rows_per_dst=[10, 19024, 12, 0, 18, 20, 0, 4] combine_rows_per_src=[16376, 16384, 16384, 12872, 1198, 16364, 16380, 16026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130348 payload_rows_remote=113964 dup_rows=724 dup_rows_remote=724 dedup_ratio=0.005524 router_tokens_per_expert=[2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 353, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7839, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16370, 16384, 16384, 15678, 16380, 16384, 16384] dup_rows_per_dst=[4, 706, 0, 0, 0, 0, 10, 4] combine_rows_per_src=[16384, 16370, 16384, 16384, 15678, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=851361792 dispatch_meta_bytes=458688 combine_payload_bytes=851361792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=851820480 ingress_bytes=926859264 link_bytes=926859264 token_bytes=1703182272 weight_movement_bytes=75497472 total_nvlink_bytes=1778679744 weight_to_token_ratio=0.044327 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.175388012e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128964 payload_rows_remote=114688 dup_rows=2108 dup_rows_remote=2108 dedup_ratio=0.016083 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1048, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7138, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 14276, 16384, 16384, 16384] dup_rows_per_dst=[0, 2096, 0, 0, 0, 0, 12, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 14276, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=783155200 dispatch_meta_bytes=382656 combine_payload_bytes=783155200 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=783537856 ingress_bytes=1085145088 link_bytes=1085145088 token_bytes=1566693056 weight_movement_bytes=301989888 total_nvlink_bytes=1868682944 weight_to_token_ratio=0.192756 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=33792 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.376116717e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130942 payload_rows_remote=114558 dup_rows=130 dup_rows_remote=130 dedup_ratio=0.000992 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16258, 16382, 16384, 16384] dup_rows_per_dst=[0, 12, 0, 0, 0, 0, 114, 4] combine_rows_per_src=[16384, 16382, 16384, 16384, 16258, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126518 payload_rows_remote=110134 dup_rows=4554 dup_rows_remote=4494 dedup_ratio=0.034744 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 2206, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6667, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8188, 0, 8190, 0] payload_rows_per_dst=[16384, 16384, 16380, 14988, 13338, 16384, 16362, 16298] dup_rows_per_dst=[60, 4470, 16, 0, 6, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 14988, 13338, 16384, 16362, 16298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=933593088 dispatch_meta_bytes=458752 combine_payload_bytes=933593088 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=934051840 ingress_bytes=1084588032 link_bytes=1084588032 token_bytes=1867644928 weight_movement_bytes=150994944 total_nvlink_bytes=2018639872 weight_to_token_ratio=0.080848 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.375410292e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 2, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118586 payload_rows_remote=102204 dup_rows=12486 dup_rows_remote=12458 dedup_ratio=0.095261 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 46, 0, 0, 0, 2150, 0, 2, 0, 7, 7326, 0, 8178, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 5, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6297, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 720, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 11, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 18, 1, 0, 0, 1, 0, 0, 8178, 0, 0, 0, 6, 0, 8151, 8021, 0, 8180, 8] payload_rows_per_dst=[16372, 4324, 16378, 16332, 16356, 16382, 16382, 16060] dup_rows_per_dst=[38, 12, 8, 10924, 1440, 24, 28, 12] combine_rows_per_src=[16372, 4324, 16378, 16332, 16356, 16382, 16382, 16060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125244 payload_rows_remote=108860 dup_rows=5828 dup_rows_remote=5814 dedup_ratio=0.044464 router_tokens_per_expert=[12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8164, 0, 1, 8174, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8174, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8192, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 5, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 8192, 0, 0, 2, 8180, 0, 0, 0, 0, 0] payload_rows_per_dst=[16352, 10616, 16384, 16376, 16382, 16384, 16366, 16384] dup_rows_per_dst=[18, 2, 0, 0, 5772, 12, 10, 14] combine_rows_per_src=[16352, 10616, 16384, 16376, 16382, 16384, 16366, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=467184 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939991280 ingress_bytes=1166016512 link_bytes=1166016512 token_bytes=1879515376 weight_movement_bytes=226492416 total_nvlink_bytes=2106007792 weight_to_token_ratio=0.120506 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478673066e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=938459136 dispatch_meta_bytes=458752 combine_payload_bytes=938459136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938917888 ingress_bytes=1013956608 link_bytes=1013956608 token_bytes=1877377024 weight_movement_bytes=75497472 total_nvlink_bytes=1952874496 weight_to_token_ratio=0.040214 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285839704e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=902217728 dispatch_meta_bytes=458512 combine_payload_bytes=902217728 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=902676240 ingress_bytes=1053212672 link_bytes=1053212672 token_bytes=1804893968 weight_movement_bytes=150994944 total_nvlink_bytes=1955888912 weight_to_token_ratio=0.083659 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.335621919e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=837255168 dispatch_meta_bytes=458648 combine_payload_bytes=837255168 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=837713816 ingress_bytes=912752640 link_bytes=912752640 token_bytes=1674968984 weight_movement_bytes=75497472 total_nvlink_bytes=1750466456 weight_to_token_ratio=0.045074 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.157498827e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=891781120 dispatch_meta_bytes=458696 combine_payload_bytes=891781120 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=892239816 ingress_bytes=1193771008 link_bytes=1193771008 token_bytes=1784020936 weight_movement_bytes=301989888 total_nvlink_bytes=2086010824 weight_to_token_ratio=0.169275 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.513869674e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130968 payload_rows_remote=114586 dup_rows=104 dup_rows_remote=104 dedup_ratio=0.000793 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 5, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 8192, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16368, 16384, 16384, 16318, 16382, 16384, 16384] dup_rows_per_dst=[0, 10, 0, 8, 4, 0, 6, 76] combine_rows_per_src=[16364, 16368, 16384, 16384, 16318, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130936 payload_rows_remote=114558 dup_rows=136 dup_rows_remote=84 dedup_ratio=0.001038 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 26, 8184, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 5, 5, 0, 0, 5, 0, 0, 8158, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8186, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 4, 1, 0, 0, 0, 20, 3, 0, 0, 0, 0, 0, 0, 0, 4, 12, 5, 8184, 0, 2, 1, 8187, 0, 0, 0, 0, 0] payload_rows_per_dst=[16346, 16378, 16384, 16334, 16376, 16374, 16362, 16382] dup_rows_per_dst=[0, 52, 24, 10, 2, 2, 12, 34] combine_rows_per_src=[16346, 16378, 16384, 16334, 16376, 16374, 16362, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114638 dup_rows=50 dup_rows_remote=14 dedup_ratio=0.000381 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8176, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16352, 16384, 16384, 16384, 16380, 16384, 16370, 16384] dup_rows_per_dst=[0, 0, 36, 12, 0, 0, 0, 2] combine_rows_per_src=[16352, 16384, 16384, 16384, 16380, 16384, 16370, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131048 payload_rows_remote=114664 dup_rows=24 dup_rows_remote=24 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8189, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 5, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16366, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 24, 0, 0, 0, 0, 0] combine_rows_per_src=[16378, 16384, 16384, 16366, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=938688512 dispatch_meta_bytes=458760 combine_payload_bytes=938688512 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939147272 ingress_bytes=1089683456 link_bytes=1089683456 token_bytes=1877835784 weight_movement_bytes=150994944 total_nvlink_bytes=2028830728 weight_to_token_ratio=0.080409 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381872007e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130960 payload_rows_remote=114576 dup_rows=112 dup_rows_remote=110 dedup_ratio=0.000854 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8139, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16278, 16384, 16384, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 110, 2, 0, 0, 0, 0] combine_rows_per_src=[16278, 16384, 16384, 16384, 16384, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128024 payload_rows_remote=111640 dup_rows=3048 dup_rows_remote=2956 dedup_ratio=0.023254 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1360, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 13618, 16382, 16234, 16254] dup_rows_per_dst=[92, 2808, 38, 98, 0, 12, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 13618, 16382, 16234, 16254] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=938459136 dispatch_meta_bytes=458568 combine_payload_bytes=938459136 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938917704 ingress_bytes=1089454080 link_bytes=1089454080 token_bytes=1877376840 weight_movement_bytes=150994944 total_nvlink_bytes=2028371784 weight_to_token_ratio=0.080429 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381581126e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939114496 dispatch_meta_bytes=458608 combine_payload_bytes=939114496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939573104 ingress_bytes=1014611968 link_bytes=1014611968 token_bytes=1878687600 weight_movement_bytes=75497472 total_nvlink_bytes=1954185072 weight_to_token_ratio=0.040186 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286670793e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130140 payload_rows_remote=113756 dup_rows=932 dup_rows_remote=908 dedup_ratio=0.007111 router_tokens_per_expert=[8, 0, 5, 0, 7, 8186, 0, 0, 0, 0, 0, 0, 0, 8147, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8192, 0, 13, 0, 0, 8187, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2] payload_rows_per_dst=[16208, 15734, 16384, 16384, 16356, 16374, 16316, 16384] dup_rows_per_dst=[12, 0, 88, 82, 76, 0, 650, 24] combine_rows_per_src=[16208, 15734, 16384, 16384, 16356, 16374, 16316, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130782 payload_rows_remote=114398 dup_rows=290 dup_rows_remote=264 dedup_ratio=0.002213 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 0, 8168, 0, 68, 8177, 0, 0, 0, 0, 2, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 11, 7, 0, 0, 0, 1, 0, 0, 0, 0, 0, 9, 0, 0, 5, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 1, 8113, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 6, 0, 7, 0, 0, 5, 12, 0, 0, 0, 0, 0, 0, 0, 11, 0, 1, 8165, 2, 1, 0, 8180, 0, 4, 0, 0, 8] payload_rows_per_dst=[16358, 16368, 16286, 16326, 16330, 16370, 16384, 16360] dup_rows_per_dst=[2, 136, 36, 0, 58, 8, 26, 24] combine_rows_per_src=[16358, 16368, 16286, 16326, 16330, 16370, 16384, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939327488 dispatch_meta_bytes=458752 combine_payload_bytes=939327488 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939786240 ingress_bytes=1014824960 link_bytes=1014824960 token_bytes=1879113728 weight_movement_bytes=75497472 total_nvlink_bytes=1954611200 weight_to_token_ratio=0.040177 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286940897e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=938606592 dispatch_meta_bytes=458744 combine_payload_bytes=938606592 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939065336 ingress_bytes=1089601536 link_bytes=1089601536 token_bytes=1877671928 weight_movement_bytes=150994944 total_nvlink_bytes=2028666872 weight_to_token_ratio=0.080416 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381768121e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=914554880 dispatch_meta_bytes=458384 combine_payload_bytes=914554880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=915013264 ingress_bytes=990052352 link_bytes=990052352 token_bytes=1829568144 weight_movement_bytes=75497472 total_nvlink_bytes=1905065616 weight_to_token_ratio=0.041265 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.255525743e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=931889152 dispatch_meta_bytes=458656 combine_payload_bytes=931889152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=932347808 ingress_bytes=1007386624 link_bytes=1007386624 token_bytes=1864236960 weight_movement_bytes=75497472 total_nvlink_bytes=1939734432 weight_to_token_ratio=0.040498 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.277508040e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=937148416 dispatch_meta_bytes=458648 combine_payload_bytes=937148416 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=937607064 ingress_bytes=1163640832 link_bytes=1163640832 token_bytes=1874755480 weight_movement_bytes=226492416 total_nvlink_bytes=2101247896 weight_to_token_ratio=0.120812 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.475660370e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130964 payload_rows_remote=114580 dup_rows=108 dup_rows_remote=108 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8146, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 8, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16292, 16384, 16384, 16384, 16382, 16384, 16370, 16384] dup_rows_per_dst=[0, 0, 86, 22, 0, 0, 0, 0] combine_rows_per_src=[16292, 16384, 16384, 16384, 16382, 16384, 16370, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114596 dup_rows=92 dup_rows_remote=76 dedup_ratio=0.000702 router_tokens_per_expert=[0, 4, 0, 2, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 8187, 8185, 1, 7, 0, 0, 0, 2, 0, 0, 5, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 5, 0, 6, 0, 1, 0, 2, 0, 0, 0, 8184, 0, 1, 0, 4, 8178, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 3, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 6, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16380, 16378, 16370, 16368, 16338, 16380] dup_rows_per_dst=[8, 16, 34, 22, 0, 6, 0, 6] combine_rows_per_src=[16382, 16384, 16380, 16378, 16370, 16368, 16338, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130216 payload_rows_remote=114062 dup_rows=856 dup_rows_remote=852 dedup_ratio=0.006531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 8192, 0, 0, 36, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 8192, 0, 0, 8165, 8077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 170] payload_rows_per_dst=[15820, 16380, 16154, 16384, 16332, 16378, 16384, 16384] dup_rows_per_dst=[72, 0, 4, 4, 2, 0, 2, 772] combine_rows_per_src=[15820, 16380, 16154, 16384, 16332, 16378, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=938639360 dispatch_meta_bytes=458752 combine_payload_bytes=938639360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939098112 ingress_bytes=1014136832 link_bytes=1014136832 token_bytes=1877737472 weight_movement_bytes=75497472 total_nvlink_bytes=1953234944 weight_to_token_ratio=0.040207 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286068254e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130708 payload_rows_remote=114328 dup_rows=364 dup_rows_remote=350 dedup_ratio=0.002777 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8189, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 144] payload_rows_per_dst=[16352, 16384, 16056, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[8, 0, 0, 0, 14, 0, 0, 342] combine_rows_per_src=[16352, 16384, 16056, 16384, 16380, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=938770432 dispatch_meta_bytes=458688 combine_payload_bytes=938770432 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939229120 ingress_bytes=1089765376 link_bytes=1089765376 token_bytes=1877999552 weight_movement_bytes=150994944 total_nvlink_bytes=2028994496 weight_to_token_ratio=0.080402 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381975893e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130654 payload_rows_remote=114270 dup_rows=418 dup_rows_remote=36 dedup_ratio=0.003189 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8177, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16022, 16384, 16384, 16348, 16376, 16372] dup_rows_per_dst=[382, 0, 0, 36, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16022, 16384, 16384, 16348, 16376, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=934395904 dispatch_meta_bytes=459656 combine_payload_bytes=934395904 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=934855560 ingress_bytes=1236385792 link_bytes=1236385792 token_bytes=1869251464 weight_movement_bytes=301989888 total_nvlink_bytes=2171241352 weight_to_token_ratio=0.161557 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=63360 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.567911219e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127934 payload_rows_remote=111560 dup_rows=3138 dup_rows_remote=3002 dedup_ratio=0.023941 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8187, 0, 0, 14, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8192, 0, 0, 32, 0, 0, 8182, 0, 0, 8174, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23] payload_rows_per_dst=[13482, 16374, 16276, 16384, 16358, 16302, 16384, 16374] dup_rows_per_dst=[28, 66, 2810, 6, 0, 62, 30, 136] combine_rows_per_src=[13482, 16374, 16276, 16384, 16358, 16302, 16384, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130326 payload_rows_remote=114436 dup_rows=746 dup_rows_remote=746 dedup_ratio=0.005692 router_tokens_per_expert=[0, 4, 0, 13, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 8181, 8178, 0, 10, 0, 0, 0, 7, 297, 0, 12, 0, 0, 9, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5, 0, 2, 0, 0, 0, 0, 0, 0, 4, 8092, 0, 0, 0, 0, 8118, 0, 0, 0, 0, 6, 14, 0, 0, 0, 0, 1, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3, 0, 0, 3, 10, 0, 0, 0, 0, 11, 60, 11, 0, 7944, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16384, 16366, 16216, 16356, 16376, 15890, 16368] dup_rows_per_dst=[26, 634, 6, 16, 0, 64, 0, 0] combine_rows_per_src=[16370, 16384, 16366, 16216, 16356, 16376, 15890, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=936574976 dispatch_meta_bytes=458712 combine_payload_bytes=936574976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937033688 ingress_bytes=1012072448 link_bytes=1012072448 token_bytes=1873608664 weight_movement_bytes=75497472 total_nvlink_bytes=1949106136 weight_to_token_ratio=0.040295 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283450324e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441408 weight_movement_bytes=75497472 total_nvlink_bytes=1954938880 weight_to_token_ratio=0.040170 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=936099840 dispatch_meta_bytes=457224 combine_payload_bytes=936099840 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=936557064 ingress_bytes=1313587200 link_bytes=1313587200 token_bytes=1872656904 weight_movement_bytes=377487360 total_nvlink_bytes=2250144264 weight_to_token_ratio=0.201578 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.665813471e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=913899520 dispatch_meta_bytes=458248 combine_payload_bytes=913899520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=914357768 ingress_bytes=989396992 link_bytes=989396992 token_bytes=1828257288 weight_movement_bytes=75497472 total_nvlink_bytes=1903754760 weight_to_token_ratio=0.041295 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=58624 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.254694654e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130772 payload_rows_remote=114406 dup_rows=300 dup_rows_remote=300 dedup_ratio=0.002289 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 8192, 0, 0, 42, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8149, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16224, 16384, 16330, 16384, 16316, 16366, 16384, 16384] dup_rows_per_dst=[80, 0, 0, 2, 6, 0, 0, 212] combine_rows_per_src=[16224, 16384, 16330, 16384, 16316, 16366, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=937459712 dispatch_meta_bytes=460728 combine_payload_bytes=937459712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937920440 ingress_bytes=1012957184 link_bytes=1012957184 token_bytes=1875380152 weight_movement_bytes=75497472 total_nvlink_bytes=1950877624 weight_to_token_ratio=0.040257 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284572294e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130814 payload_rows_remote=114484 dup_rows=258 dup_rows_remote=242 dedup_ratio=0.001968 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 8, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8, 0, 0, 0, 0, 0, 8184, 0, 0, 32, 0, 4, 8164, 0, 0, 8120, 8180, 0, 0, 0, 0, 5, 5, 3, 3, 0, 0, 0, 0, 0, 3, 10, 2, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 113, 0, 0, 0, 0, 0, 0, 0, 0, 10, 7, 0, 0, 0, 8180, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[16264, 16330, 16368, 16364, 16358, 16384, 16370, 16376] dup_rows_per_dst=[0, 16, 64, 26, 122, 22, 2, 6] combine_rows_per_src=[16264, 16330, 16368, 16364, 16358, 16384, 16370, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124396 payload_rows_remote=108012 dup_rows=6676 dup_rows_remote=6672 dedup_ratio=0.050934 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 48, 8192, 0, 25, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8192, 7414, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 804, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 8171, 0] payload_rows_per_dst=[16384, 16384, 16384, 16358, 16260, 16380, 9910, 16336] dup_rows_per_dst=[0, 146, 4, 0, 0, 6520, 6, 0] combine_rows_per_src=[16384, 16384, 16384, 16358, 16260, 16380, 9910, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=937213952 dispatch_meta_bytes=458824 combine_payload_bytes=937213952 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937672776 ingress_bytes=1012711424 link_bytes=1012711424 token_bytes=1874886728 weight_movement_bytes=75497472 total_nvlink_bytes=1950384200 weight_to_token_ratio=0.040268 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284260636e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130270 payload_rows_remote=114650 dup_rows=802 dup_rows_remote=802 dedup_ratio=0.006119 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7810, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 361, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15620, 16358, 16384, 16372] dup_rows_per_dst=[0, 80, 0, 0, 0, 0, 722, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 15620, 16358, 16384, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120176 payload_rows_remote=103806 dup_rows=10896 dup_rows_remote=10896 dedup_ratio=0.083130 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 41, 8192, 0, 9, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 2812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5398, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 8184, 0] payload_rows_per_dst=[16384, 16384, 16384, 16370, 16276, 5624, 16384, 16370] dup_rows_per_dst=[0, 100, 0, 0, 0, 0, 10796, 0] combine_rows_per_src=[16384, 16384, 16384, 16370, 16276, 5624, 16384, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=937852928 dispatch_meta_bytes=458904 combine_payload_bytes=937852928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938311832 ingress_bytes=1013350400 link_bytes=1013350400 token_bytes=1876164760 weight_movement_bytes=75497472 total_nvlink_bytes=1951662232 weight_to_token_ratio=0.040240 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285070947e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119604 payload_rows_remote=104190 dup_rows=11468 dup_rows_remote=2682 dedup_ratio=0.087494 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 5185, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8184, 6377, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2607, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2196] payload_rows_per_dst=[15414, 16384, 13298, 16322, 14950, 16384, 16380, 10472] dup_rows_per_dst=[8786, 0, 994, 0, 0, 0, 0, 1688] combine_rows_per_src=[15414, 16384, 13298, 16322, 14950, 16384, 16380, 10472] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=884834304 dispatch_meta_bytes=458736 combine_payload_bytes=884834304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=885293040 ingress_bytes=960331776 link_bytes=960331776 token_bytes=1770127344 weight_movement_bytes=75497472 total_nvlink_bytes=1845624816 weight_to_token_ratio=0.042651 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.217835869e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939212800 dispatch_meta_bytes=461808 combine_payload_bytes=939212800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939674608 ingress_bytes=1014710272 link_bytes=1014710272 token_bytes=1878887408 weight_movement_bytes=75497472 total_nvlink_bytes=1954384880 weight_to_token_ratio=0.040182 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286795456e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125156 payload_rows_remote=109032 dup_rows=5916 dup_rows_remote=5426 dedup_ratio=0.045135 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 125, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 27, 0, 0, 0, 5, 0, 0, 0, 8175, 0, 0, 8187, 6035, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2276, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8062, 0, 0, 8136, 0] payload_rows_per_dst=[16374, 15316, 16384, 16330, 16138, 12106, 16384, 16124] dup_rows_per_dst=[0, 250, 52, 0, 54, 552, 4518, 490] combine_rows_per_src=[16374, 15316, 16384, 16330, 16138, 12106, 16384, 16124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=850378752 dispatch_meta_bytes=458808 combine_payload_bytes=850378752 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=850837560 ingress_bytes=1001373696 link_bytes=1001373696 token_bytes=1701216312 weight_movement_bytes=150994944 total_nvlink_bytes=1852211256 weight_to_token_ratio=0.088757 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.269882801e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126660 payload_rows_remote=110302 dup_rows=4412 dup_rows_remote=4402 dedup_ratio=0.033661 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 6039, 0, 0, 0, 8174, 12, 0, 0, 27, 0, 0, 0, 0, 8175, 6, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 2, 0, 0, 0, 4, 8161, 0, 0, 0, 0, 8, 8155, 11, 0, 8170, 8143, 0, 0, 0, 0, 12, 0, 14, 12, 0, 21, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 294, 4, 0, 0, 0, 0, 449, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 13, 8178, 0, 0, 10, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 16, 0, 0, 0, 0, 0, 1386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19] payload_rows_per_dst=[12148, 16366, 16324, 16380, 16360, 16376, 16358, 16348] dup_rows_per_dst=[10, 4, 12, 48, 1486, 32, 10, 2810] combine_rows_per_src=[12148, 16366, 16324, 16380, 16360, 16376, 16358, 16348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=853524480 dispatch_meta_bytes=427488 combine_payload_bytes=853524480 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=853951968 ingress_bytes=1155514368 link_bytes=1155514368 token_bytes=1707476448 weight_movement_bytes=301989888 total_nvlink_bytes=2009466336 weight_to_token_ratio=0.176863 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=58496 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.465354870e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125938 payload_rows_remote=114340 dup_rows=5134 dup_rows_remote=5134 dedup_ratio=0.039169 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 8192, 0, 65, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 5799, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 8170, 0] payload_rows_per_dst=[16384, 16384, 16384, 16376, 16118, 11598, 16384, 16310] dup_rows_per_dst=[0, 228, 4, 0, 0, 0, 4902, 0] combine_rows_per_src=[16384, 16384, 16384, 16376, 16118, 11598, 16384, 16310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=903593984 dispatch_meta_bytes=458816 combine_payload_bytes=903593984 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=904052800 ingress_bytes=979091456 link_bytes=979091456 token_bytes=1807646784 weight_movement_bytes=75497472 total_nvlink_bytes=1883144256 weight_to_token_ratio=0.041766 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.241625784e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=893190144 dispatch_meta_bytes=457832 combine_payload_bytes=893190144 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=893647976 ingress_bytes=1119682560 link_bytes=1119682560 token_bytes=1786838120 weight_movement_bytes=226492416 total_nvlink_bytes=2013330536 weight_to_token_ratio=0.126756 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.419915093e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130840 payload_rows_remote=114470 dup_rows=232 dup_rows_remote=140 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 46, 0, 0, 1, 0, 0, 0, 8184, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 10, 17, 0, 0, 5, 0, 6, 0, 0, 4, 0, 0, 0, 4, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8172, 0, 5, 8183, 31, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 8151, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8171, 0, 0, 8131, 0] payload_rows_per_dst=[16372, 16370, 16376, 16310, 16384, 16360, 16314, 16354] dup_rows_per_dst=[4, 92, 10, 34, 8, 70, 14, 0] combine_rows_per_src=[16372, 16370, 16376, 16310, 16384, 16360, 16314, 16354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=936673280 dispatch_meta_bytes=477896 combine_payload_bytes=936673280 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=937151176 ingress_bytes=1314160640 link_bytes=1314160640 token_bytes=1873824456 weight_movement_bytes=377487360 total_nvlink_bytes=2251311816 weight_to_token_ratio=0.201453 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=44928 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.666540674e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=20 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16376, 16384, 16384, 16360, 16382] dup_rows_per_dst=[0, 8, 20, 6, 2, 0, 4, 0] combine_rows_per_src=[16378, 16384, 16384, 16376, 16384, 16384, 16360, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131042 payload_rows_remote=114682 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000229 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16360, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 4, 10, 0, 0, 0, 16, 0] combine_rows_per_src=[16384, 16384, 16384, 16360, 16384, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=937738240 dispatch_meta_bytes=458440 combine_payload_bytes=937738240 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=938196680 ingress_bytes=1164230656 link_bytes=1164230656 token_bytes=1875934920 weight_movement_bytes=226492416 total_nvlink_bytes=2102427336 weight_to_token_ratio=0.120736 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.476408350e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115654 payload_rows_remote=99270 dup_rows=15418 dup_rows_remote=15398 dedup_ratio=0.117630 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 7733, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 781, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16334, 16384, 15886, 16384, 1562, 16336] dup_rows_per_dst=[20, 70, 0, 50, 2198, 13080, 0, 0] combine_rows_per_src=[16384, 16384, 16334, 16384, 15886, 16384, 1562, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458672 combine_payload_bytes=939196416 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939655088 ingress_bytes=1165688832 link_bytes=1165688832 token_bytes=1878851504 weight_movement_bytes=226492416 total_nvlink_bytes=2105343920 weight_to_token_ratio=0.120548 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478257522e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939950080 ingress_bytes=1090486272 link_bytes=1090486272 token_bytes=1879441408 weight_movement_bytes=150994944 total_nvlink_bytes=2030436352 weight_to_token_ratio=0.080340 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382890090e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130374 payload_rows_remote=114008 dup_rows=698 dup_rows_remote=272 dedup_ratio=0.005325 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 77, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 7, 16, 0, 0, 51, 0, 0, 0, 12, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 2, 8169, 7992, 0, 0, 7, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 201, 3, 0, 0, 0, 40, 0, 4, 0, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 8175, 0, 0, 8146, 0] payload_rows_per_dst=[16372, 16346, 16366, 16384, 16130, 16030, 16366, 16380] dup_rows_per_dst=[10, 154, 6, 92, 0, 10, 426, 0] combine_rows_per_src=[16372, 16346, 16366, 16384, 16130, 16030, 16366, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130272 payload_rows_remote=113904 dup_rows=800 dup_rows_remote=800 dedup_ratio=0.006104 router_tokens_per_expert=[0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 8192, 0, 0, 45, 0, 0, 0, 0, 0, 10, 0, 0, 3, 0, 0, 0, 0, 16, 0, 0, 8178, 0, 0, 0, 8189, 0, 0, 0, 36, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 8, 0, 0, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16348, 15920, 16262, 16306, 16384, 16378, 16306, 16368] dup_rows_per_dst=[0, 116, 78, 2, 124, 0, 480, 0] combine_rows_per_src=[16348, 15920, 16262, 16306, 16384, 16378, 16306, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458848 combine_payload_bytes=939474944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939933792 ingress_bytes=1014972416 link_bytes=1014972416 token_bytes=1879408736 weight_movement_bytes=75497472 total_nvlink_bytes=1954906208 weight_to_token_ratio=0.040171 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287127892e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=813219840 dispatch_meta_bytes=458672 combine_payload_bytes=813219840 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=813678512 ingress_bytes=964214784 link_bytes=964214784 token_bytes=1626898352 weight_movement_bytes=150994944 total_nvlink_bytes=1777893296 weight_to_token_ratio=0.092812 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.222760070e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131036 payload_rows_remote=114654 dup_rows=36 dup_rows_remote=36 dedup_ratio=0.000275 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16354, 16384, 16382, 16380, 16384] dup_rows_per_dst=[0, 8, 0, 0, 18, 0, 10, 0] combine_rows_per_src=[16384, 16384, 16384, 16354, 16384, 16382, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=933953536 dispatch_meta_bytes=457120 combine_payload_bytes=933953536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=934410656 ingress_bytes=1009451008 link_bytes=1009451008 token_bytes=1868364192 weight_movement_bytes=75497472 total_nvlink_bytes=1943861664 weight_to_token_ratio=0.040408 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=45056 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.280125969e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=933101568 dispatch_meta_bytes=458816 combine_payload_bytes=933101568 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=933560384 ingress_bytes=1084096512 link_bytes=1084096512 token_bytes=1866661952 weight_movement_bytes=150994944 total_nvlink_bytes=2017656896 weight_to_token_ratio=0.080890 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.374786975e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130650 payload_rows_remote=114266 dup_rows=422 dup_rows_remote=332 dedup_ratio=0.003220 router_tokens_per_expert=[0, 0, 0, 0, 0, 8139, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8175, 8179, 0, 0, 38, 15, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 8186, 0, 0, 2, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 5, 0, 3, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5, 0, 6, 0, 7, 0, 0, 0, 1, 0, 0, 0, 4, 0, 8045, 0, 0, 0, 8, 0, 0, 9, 0, 0, 0, 0, 0, 8184, 4, 0, 4, 0, 0, 0] payload_rows_per_dst=[16282, 16384, 16346, 16374, 16370, 16384, 16126, 16384] dup_rows_per_dst=[0, 90, 0, 212, 70, 16, 0, 34] combine_rows_per_src=[16282, 16384, 16346, 16374, 16370, 16384, 16126, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[2, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114682 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 6] combine_rows_per_src=[16378, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=939245568 dispatch_meta_bytes=458760 combine_payload_bytes=939245568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939704328 ingress_bytes=1014743040 link_bytes=1014743040 token_bytes=1878949896 weight_movement_bytes=75497472 total_nvlink_bytes=1954447368 weight_to_token_ratio=0.040181 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286837011e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130654 payload_rows_remote=114270 dup_rows=418 dup_rows_remote=326 dedup_ratio=0.003189 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16316, 16148, 16306, 16384, 16348] dup_rows_per_dst=[92, 236, 90, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16316, 16148, 16306, 16384, 16348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=936067072 dispatch_meta_bytes=458392 combine_payload_bytes=936067072 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=936525464 ingress_bytes=1087062016 link_bytes=1087062016 token_bytes=1872592536 weight_movement_bytes=150994944 total_nvlink_bytes=2023587480 weight_to_token_ratio=0.080634 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378547652e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=939966464 ingress_bytes=1166000128 link_bytes=1166000128 token_bytes=1879474176 weight_movement_bytes=226492416 total_nvlink_bytes=2105966592 weight_to_token_ratio=0.120508 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478652289e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458752 combine_payload_bytes=939474944 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939933696 ingress_bytes=1165967360 link_bytes=1165967360 token_bytes=1879408640 weight_movement_bytes=226492416 total_nvlink_bytes=2105901056 weight_to_token_ratio=0.120513 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478610735e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441408 weight_movement_bytes=75497472 total_nvlink_bytes=1954938880 weight_to_token_ratio=0.040170 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130794 payload_rows_remote=114568 dup_rows=278 dup_rows_remote=264 dedup_ratio=0.002121 router_tokens_per_expert=[0, 99, 29, 0, 0, 8068, 0, 0, 0, 0, 0, 22, 0, 0, 0, 8173, 8136, 0, 0, 0, 49, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 8177, 0, 0, 3, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 3, 9, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 1, 0, 15, 0, 0, 0, 0, 0, 0, 20, 0, 14, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7, 0, 8096, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 15, 0, 0, 0, 0, 0] payload_rows_per_dst=[16366, 16382, 16346, 16358, 16350, 16384, 16226, 16382] dup_rows_per_dst=[70, 4, 0, 24, 70, 74, 14, 22] combine_rows_per_src=[16366, 16382, 16346, 16358, 16350, 16384, 16226, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130120 payload_rows_remote=113736 dup_rows=952 dup_rows_remote=66 dedup_ratio=0.007263 router_tokens_per_expert=[0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 9, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 6, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 51, 0, 0, 21, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 8182, 0, 0] payload_rows_per_dst=[16322, 16260, 15706, 16368, 16312, 16384, 16384, 16384] dup_rows_per_dst=[0, 10, 12, 0, 0, 26, 18, 886] combine_rows_per_src=[16322, 16260, 15706, 16368, 16312, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=936099840 dispatch_meta_bytes=458384 combine_payload_bytes=936099840 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=936558224 ingress_bytes=1087094784 link_bytes=1087094784 token_bytes=1872658064 weight_movement_bytes=150994944 total_nvlink_bytes=2023653008 weight_to_token_ratio=0.080631 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378589206e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131018 payload_rows_remote=114634 dup_rows=54 dup_rows_remote=54 dedup_ratio=0.000412 router_tokens_per_expert=[0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16378, 16384, 16340, 16384] dup_rows_per_dst=[2, 44, 0, 0, 0, 0, 0, 8] combine_rows_per_src=[16380, 16384, 16384, 16384, 16378, 16384, 16340, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938541056 dispatch_meta_bytes=459328 combine_payload_bytes=938541056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939000384 ingress_bytes=1014038528 link_bytes=1014038528 token_bytes=1877541440 weight_movement_bytes=75497472 total_nvlink_bytes=1953038912 weight_to_token_ratio=0.040211 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285943590e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=931725312 dispatch_meta_bytes=455208 combine_payload_bytes=931725312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=932180520 ingress_bytes=1007222784 link_bytes=1007222784 token_bytes=1863905832 weight_movement_bytes=75497472 total_nvlink_bytes=1939403304 weight_to_token_ratio=0.040505 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.277300267e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130650 payload_rows_remote=114266 dup_rows=422 dup_rows_remote=404 dedup_ratio=0.003220 router_tokens_per_expert=[0, 0, 0, 0, 2, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 8181, 10, 3, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 8, 8181, 0, 0, 147, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8179, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 53, 0, 0, 0, 8149, 7, 0, 0, 0, 0, 0, 10, 0, 0, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 8178, 0, 1, 0, 3, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 5, 5, 0, 0, 0, 0, 2, 8003, 0, 0] payload_rows_per_dst=[16370, 16384, 16356, 16380, 16380, 16374, 16376, 16030] dup_rows_per_dst=[12, 18, 294, 4, 64, 16, 10, 4] combine_rows_per_src=[16370, 16384, 16356, 16380, 16380, 16374, 16376, 16030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131042 payload_rows_remote=114658 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000229 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16354, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 30, 0, 0] combine_rows_per_src=[16384, 16384, 16354, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939081728 dispatch_meta_bytes=458752 combine_payload_bytes=939081728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939540480 ingress_bytes=1014579200 link_bytes=1014579200 token_bytes=1878622208 weight_movement_bytes=75497472 total_nvlink_bytes=1954119680 weight_to_token_ratio=0.040188 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286629238e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130816 payload_rows_remote=114436 dup_rows=256 dup_rows_remote=256 dedup_ratio=0.001953 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 55, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8074, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 10, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0] payload_rows_per_dst=[16384, 16368, 16380, 16148, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 108, 0, 0, 100, 10, 20, 18] combine_rows_per_src=[16384, 16368, 16380, 16148, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130892 payload_rows_remote=114682 dup_rows=180 dup_rows_remote=180 dedup_ratio=0.001373 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8105, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16378, 16210, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 156, 10, 14, 0] combine_rows_per_src=[16384, 16384, 16378, 16210, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114596 dup_rows=92 dup_rows_remote=88 dedup_ratio=0.000702 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16292, 16384, 16384, 16384] dup_rows_per_dst=[4, 0, 42, 46, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16292, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=936067072 dispatch_meta_bytes=458680 combine_payload_bytes=936067072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=936525752 ingress_bytes=1011564544 link_bytes=1011564544 token_bytes=1872592824 weight_movement_bytes=75497472 total_nvlink_bytes=1948090296 weight_to_token_ratio=0.040317 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.282806230e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939278336 dispatch_meta_bytes=458752 combine_payload_bytes=939278336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939737088 ingress_bytes=1014775808 link_bytes=1014775808 token_bytes=1879015424 weight_movement_bytes=75497472 total_nvlink_bytes=1954512896 weight_to_token_ratio=0.040179 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286878565e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=937459712 dispatch_meta_bytes=458768 combine_payload_bytes=937459712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937918480 ingress_bytes=1012957184 link_bytes=1012957184 token_bytes=1875378192 weight_movement_bytes=75497472 total_nvlink_bytes=1950875664 weight_to_token_ratio=0.040257 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284572294e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130318 payload_rows_remote=113950 dup_rows=754 dup_rows_remote=710 dedup_ratio=0.005753 router_tokens_per_expert=[0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 33, 2, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 1, 0, 0, 0, 13, 0, 5, 8139, 0, 8167, 0, 0, 214, 0, 0, 0, 0, 0, 0, 0, 13, 2, 8160, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 102, 0, 0, 0, 8132, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 17, 0, 0, 11, 0, 0, 0, 4, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 8169, 13, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 14, 0, 0, 0, 0, 0, 7883, 0, 0] payload_rows_per_dst=[16354, 16384, 16332, 16346, 16360, 16344, 16368, 15830] dup_rows_per_dst=[98, 48, 410, 8, 134, 8, 44, 4] combine_rows_per_src=[16354, 16384, 16332, 16346, 16360, 16344, 16368, 15830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939474944 dispatch_meta_bytes=459448 combine_payload_bytes=939474944 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939934392 ingress_bytes=1241464832 link_bytes=1241464832 token_bytes=1879409336 weight_movement_bytes=301989888 total_nvlink_bytes=2181399224 weight_to_token_ratio=0.160683 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.045956 gemm_active_groups=12 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.574352156e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=938770432 dispatch_meta_bytes=458736 combine_payload_bytes=938770432 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939229168 ingress_bytes=1165262848 link_bytes=1165262848 token_bytes=1877999600 weight_movement_bytes=226492416 total_nvlink_bytes=2104492016 weight_to_token_ratio=0.120603 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.477717314e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127506 payload_rows_remote=111122 dup_rows=3566 dup_rows_remote=3216 dedup_ratio=0.027206 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 306, 0, 4, 0, 0, 8192, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7769, 2, 0, 8093, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 100, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 44, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 177, 0, 0] payload_rows_per_dst=[16384, 13720, 16328, 15544, 16378, 16384, 16384, 16384] dup_rows_per_dst=[0, 504, 0, 2470, 8, 190, 44, 350] combine_rows_per_src=[16384, 13720, 16328, 15544, 16378, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130654 payload_rows_remote=114270 dup_rows=418 dup_rows_remote=416 dedup_ratio=0.003189 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7999, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 134, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 40, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0] payload_rows_per_dst=[16384, 16352, 16384, 15998, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 118, 0, 0, 188, 2, 80, 30] combine_rows_per_src=[16384, 16352, 16384, 15998, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=933478400 dispatch_meta_bytes=458640 combine_payload_bytes=933478400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933937040 ingress_bytes=1008975872 link_bytes=1008975872 token_bytes=1867415440 weight_movement_bytes=75497472 total_nvlink_bytes=1942912912 weight_to_token_ratio=0.040429 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.279523430e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=910311424 dispatch_meta_bytes=457352 combine_payload_bytes=910311424 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=910768776 ingress_bytes=985808896 link_bytes=985808896 token_bytes=1821080200 weight_movement_bytes=75497472 total_nvlink_bytes=1896577672 weight_to_token_ratio=0.041458 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.250144443e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=936099840 dispatch_meta_bytes=458744 combine_payload_bytes=936099840 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=936558584 ingress_bytes=1087094784 link_bytes=1087094784 token_bytes=1872658424 weight_movement_bytes=150994944 total_nvlink_bytes=2023653368 weight_to_token_ratio=0.080631 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378589206e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129424 payload_rows_remote=113076 dup_rows=1648 dup_rows_remote=780 dedup_ratio=0.012573 router_tokens_per_expert=[0, 4, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 517, 0, 0, 0, 0, 8136, 0, 0, 0, 8, 8083, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8171, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 34, 8, 5, 0, 9, 1, 0, 0, 7426, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 6, 0, 0, 0, 7, 0, 0, 0, 4, 0, 0, 0, 0, 14, 0, 3, 0, 3, 0, 0, 0, 8178, 0, 196, 0, 19, 0, 0, 0, 3, 0, 8165, 0, 0, 4, 0, 3, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 148, 0, 0] payload_rows_per_dst=[16364, 16348, 16344, 14926, 16372, 16310, 16378, 16382] dup_rows_per_dst=[8, 868, 4, 40, 18, 10, 402, 298] combine_rows_per_src=[16364, 16348, 16344, 14926, 16372, 16310, 16378, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130890 payload_rows_remote=114506 dup_rows=182 dup_rows_remote=182 dedup_ratio=0.001389 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8107, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 21, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16302, 16284, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 182, 0, 0] combine_rows_per_src=[16384, 16384, 16302, 16284, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125254 payload_rows_remote=109726 dup_rows=5818 dup_rows_remote=5792 dedup_ratio=0.044388 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 7654, 0, 0, 8192, 0, 13, 0, 0, 8192, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7540, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 1076, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16384, 15528, 15124, 16384, 12696, 16380, 16384] dup_rows_per_dst=[0, 14, 26, 4504, 24, 1250, 0, 0] combine_rows_per_src=[16374, 16384, 15528, 15124, 16384, 12696, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129250 payload_rows_remote=114152 dup_rows=1822 dup_rows_remote=1822 dedup_ratio=0.013901 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7704, 0, 0, 8192, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7549, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 907, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 15856, 15098, 16384, 16380, 16382, 16384] dup_rows_per_dst=[0, 6, 2, 0, 0, 1814, 0, 0] combine_rows_per_src=[16382, 16384, 15856, 15098, 16384, 16380, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129562 payload_rows_remote=113178 dup_rows=1510 dup_rows_remote=1476 dedup_ratio=0.011520 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 48, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16366, 16378, 15924, 16384, 16008, 15734] dup_rows_per_dst=[34, 932, 94, 350, 0, 94, 6, 0] combine_rows_per_src=[16384, 16384, 16366, 16378, 15924, 16384, 16008, 15734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 2, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=926318592 dispatch_meta_bytes=455424 combine_payload_bytes=926318592 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=926774016 ingress_bytes=1228308480 link_bytes=1228308480 token_bytes=1853092608 weight_movement_bytes=301989888 total_nvlink_bytes=2155082496 weight_to_token_ratio=0.162965 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=64000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.557668050e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938033152 dispatch_meta_bytes=458752 combine_payload_bytes=938033152 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938491904 ingress_bytes=1089028096 link_bytes=1089028096 token_bytes=1876525056 weight_movement_bytes=150994944 total_nvlink_bytes=2027520000 weight_to_token_ratio=0.080465 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381040918e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130468 payload_rows_remote=114288 dup_rows=604 dup_rows_remote=584 dedup_ratio=0.004608 router_tokens_per_expert=[0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8035, 0, 8, 10, 15, 8127, 77, 61, 0, 25, 0, 21, 0, 0, 1, 8147, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 8165, 0, 0, 19, 0, 128, 0, 20, 0, 5, 8140, 0, 0, 8174, 0, 0, 0, 0, 11, 0, 8, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 11, 0, 0, 0, 4, 8170, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8081, 0, 0, 9, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16360, 16362, 16374, 16384, 16372, 16096, 16180, 16340] dup_rows_per_dst=[2, 240, 66, 240, 20, 16, 20, 0] combine_rows_per_src=[16360, 16362, 16374, 16384, 16372, 16096, 16180, 16340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=898875392 dispatch_meta_bytes=462072 combine_payload_bytes=898875392 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=899337464 ingress_bytes=1351860224 link_bytes=1351860224 token_bytes=1798212856 weight_movement_bytes=452984832 total_nvlink_bytes=2251197688 weight_to_token_ratio=0.251908 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.051471 gemm_active_groups=9 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.714349053e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=935133184 dispatch_meta_bytes=463896 combine_payload_bytes=935133184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935597080 ingress_bytes=1010630656 link_bytes=1010630656 token_bytes=1870730264 weight_movement_bytes=75497472 total_nvlink_bytes=1946227736 weight_to_token_ratio=0.040357 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=48384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281621929e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=927154176 dispatch_meta_bytes=458616 combine_payload_bytes=927154176 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=927612792 ingress_bytes=1078149120 link_bytes=1078149120 token_bytes=1854766968 weight_movement_bytes=150994944 total_nvlink_bytes=2005761912 weight_to_token_ratio=0.081409 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.367244845e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124230 payload_rows_remote=107846 dup_rows=6842 dup_rows_remote=6768 dedup_ratio=0.052200 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 29, 0, 61, 0, 33, 0, 0, 25, 0, 0, 0, 0, 0, 8158, 0, 7781, 0, 0, 8192, 0, 86, 0, 0, 8103, 4, 0, 0, 0, 404, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5724, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 2424, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0] payload_rows_per_dst=[16372, 14690, 16372, 11448, 16384, 16264, 16316, 16384] dup_rows_per_dst=[0, 142, 1504, 0, 0, 4848, 274, 74] combine_rows_per_src=[16372, 14690, 16372, 11448, 16384, 16264, 16316, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125348 payload_rows_remote=109006 dup_rows=5724 dup_rows_remote=58 dedup_ratio=0.043671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 7576, 0, 0, 8192, 0, 14, 0, 0, 8187, 0, 0, 0, 12, 310, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5652, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 2833, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 15796, 11304, 16384, 16342, 16374, 16384] dup_rows_per_dst=[0, 30, 28, 0, 0, 5666, 0, 0] combine_rows_per_src=[16380, 16384, 15796, 11304, 16384, 16342, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21907483816 prefetch_bytes=905969664 reduce_bytes=528482304 weight_movement_bytes=1434451968 egress_bytes=10956483752 ingress_bytes=12385452032 link_bytes=12385452032 total_nvlink_bytes=23341935784 weight_to_token_ratio=0.065478 prefetch_share_of_ingress=0.073148 dedup_ratio=0.028624 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=7 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048100 gemm_inner_pad_upper_bound=12446 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.308874713e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=936247296 dispatch_meta_bytes=459488 combine_payload_bytes=936247296 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=936706784 ingress_bytes=1162739712 link_bytes=1162739712 token_bytes=1872954080 weight_movement_bytes=226492416 total_nvlink_bytes=2099446496 weight_to_token_ratio=0.120928 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.474517623e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21569962056 prefetch_bytes=1056964608 reduce_bytes=1056964608 weight_movement_bytes=2113929216 egress_bytes=10787700808 ingress_bytes=12896190464 link_bytes=12896190464 total_nvlink_bytes=23683891272 weight_to_token_ratio=0.098003 prefetch_share_of_ingress=0.081959 dedup_ratio=0.040474 prefetch_slots=28 unused_prefetch_slots=356 slot_utilization=0.072917 reduce_slots=14 remote_demand=14 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047947 gemm_inner_pad_upper_bound=13208 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.362848732e-04 prefetch_bytes_per_gemm_flop=1.116983252e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21910390480 prefetch_bytes=981467136 reduce_bytes=528482304 weight_movement_bytes=1509949440 egress_bytes=10957965008 ingress_bytes=12462374912 link_bytes=12462374912 total_nvlink_bytes=23420339920 weight_to_token_ratio=0.068915 prefetch_share_of_ingress=0.078754 dedup_ratio=0.031916 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=7 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.046262 gemm_inner_pad_upper_bound=14605 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.317003800e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=883474432 dispatch_meta_bytes=458456 combine_payload_bytes=883474432 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=883932888 ingress_bytes=1034469376 link_bytes=1034469376 token_bytes=1767407320 weight_movement_bytes=150994944 total_nvlink_bytes=1918402264 weight_to_token_ratio=0.085433 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.311852781e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=892977152 dispatch_meta_bytes=436256 combine_payload_bytes=892977152 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=893413408 ingress_bytes=1119469568 link_bytes=1119469568 token_bytes=1786390560 weight_movement_bytes=226492416 total_nvlink_bytes=2012882976 weight_to_token_ratio=0.126788 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=46848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.419644989e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115316 payload_rows_remote=98940 dup_rows=15756 dup_rows_remote=14400 dedup_ratio=0.120209 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 8181, 0, 0, 0, 0, 1, 0, 0, 774, 46, 0, 0, 4, 6, 0, 0, 3, 0, 3, 0, 0, 8132, 0, 7149, 0, 0, 8182, 0, 0, 0, 0, 8029, 0, 9, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 18, 0, 0, 7166, 0, 0, 0, 0, 0, 0, 8, 7, 0, 0, 0, 1, 0, 0, 5, 0, 0, 8177, 0, 8148, 8, 0, 7, 0, 0, 0, 0, 1418, 0, 16, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 3, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16376, 14316, 16336, 16372, 2862, 16304, 16374] dup_rows_per_dst=[0, 1356, 0, 14332, 24, 36, 6, 2] combine_rows_per_src=[16376, 16376, 14316, 16336, 16372, 2862, 16304, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21315291240 prefetch_bytes=754974720 reduce_bytes=1207959552 weight_movement_bytes=1962934272 egress_bytes=10660382824 ingress_bytes=12617842688 link_bytes=12617842688 total_nvlink_bytes=23278225512 weight_to_token_ratio=0.092090 prefetch_share_of_ingress=0.059834 dedup_ratio=0.049321 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=16 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047488 gemm_inner_pad_upper_bound=12954 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.333433385e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119726 payload_rows_remote=103342 dup_rows=11346 dup_rows_remote=11278 dedup_ratio=0.086563 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7849, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2626, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16196, 16316, 5294, 16384, 16384] dup_rows_per_dst=[68, 0, 2526, 8752, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16196, 16316, 5294, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21588307152 prefetch_bytes=905969664 reduce_bytes=603979776 weight_movement_bytes=1509949440 egress_bytes=10796903632 ingress_bytes=12301352960 link_bytes=12301352960 total_nvlink_bytes=23098256592 weight_to_token_ratio=0.069943 prefetch_share_of_ingress=0.073648 dedup_ratio=0.041495 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=8 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047947 gemm_inner_pad_upper_bound=12446 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.299987258e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=810516480 dispatch_meta_bytes=453360 combine_payload_bytes=810516480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=810969840 ingress_bytes=886013952 link_bytes=886013952 token_bytes=1621486320 weight_movement_bytes=75497472 total_nvlink_bytes=1696983792 weight_to_token_ratio=0.046561 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.045956 gemm_active_groups=11 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.123590407e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116024 payload_rows_remote=99768 dup_rows=15048 dup_rows_remote=15036 dedup_ratio=0.114807 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 7, 8160, 0, 0, 0, 0, 36, 0, 0, 365, 0, 0, 0, 0, 0, 0, 0, 8, 0, 6, 0, 0, 8117, 0, 8169, 0, 0, 8150, 0, 0, 0, 0, 7854, 0, 0, 0, 47, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 4, 2, 0, 799, 4, 16, 0, 4, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 11, 7, 8169, 0, 8170, 19, 0, 10, 0, 0, 0, 0, 7328, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16338, 16368, 16338, 1646, 16382, 16378, 16256, 16318] dup_rows_per_dst=[14, 170, 94, 48, 34, 14676, 12, 0] combine_rows_per_src=[16338, 16368, 16338, 1646, 16382, 16378, 16256, 16318] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=846577664 dispatch_meta_bytes=458480 combine_payload_bytes=846577664 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=847036144 ingress_bytes=922075136 link_bytes=922075136 token_bytes=1693613808 weight_movement_bytes=75497472 total_nvlink_bytes=1769111280 weight_to_token_ratio=0.044578 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.169321064e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21456684896 prefetch_bytes=830472192 reduce_bytes=1509949440 weight_movement_bytes=2340421632 egress_bytes=10731046752 ingress_bytes=13066059776 link_bytes=13066059776 total_nvlink_bytes=23797106528 weight_to_token_ratio=0.109077 prefetch_share_of_ingress=0.063559 dedup_ratio=0.045024 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=20 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047488 gemm_inner_pad_upper_bound=13843 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.380800249e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=817299456 dispatch_meta_bytes=459216 combine_payload_bytes=817299456 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=817758672 ingress_bytes=968294400 link_bytes=968294400 token_bytes=1635058128 weight_movement_bytes=150994944 total_nvlink_bytes=1786053072 weight_to_token_ratio=0.092348 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.227933597e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20637535632 prefetch_bytes=981467136 reduce_bytes=1207959552 weight_movement_bytes=2189426688 egress_bytes=10321464720 ingress_bytes=12505497600 link_bytes=12505497600 total_nvlink_bytes=22826962320 weight_to_token_ratio=0.106090 prefetch_share_of_ingress=0.078483 dedup_ratio=0.075499 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=16 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047947 gemm_inner_pad_upper_bound=12573 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.321560937e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21391547960 prefetch_bytes=754974720 reduce_bytes=528482304 weight_movement_bytes=1283457024 egress_bytes=10698513976 ingress_bytes=11976491008 link_bytes=11976491008 total_nvlink_bytes=22675004984 weight_to_token_ratio=0.059998 prefetch_share_of_ingress=0.063038 dedup_ratio=0.046786 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=7 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047947 gemm_inner_pad_upper_bound=12827 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.265656368e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none + [2026-08-29 23:35:11] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 369.0 | throughput per GPU (TFLOP/s/GPU): 490.0 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.042955E+01 | loss scale: 1.0 | grad norm: 51.173 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109632 payload_rows_remote=97108 dup_rows=21440 dup_rows_remote=20718 dedup_ratio=0.163574 router_tokens_per_expert=[0, 0, 0, 6, 9, 2, 0, 0, 0, 8175, 0, 0, 0, 2, 0, 1, 0, 3, 0, 0, 416, 0, 0, 832, 6825, 0, 0, 8191, 11, 794, 0, 0, 0, 0, 1, 0, 1, 28, 0, 1, 0, 0, 0, 406, 0, 0, 0, 6888, 0, 3, 0, 1, 0, 0, 1, 0, 1, 0, 0, 3129, 0, 0, 0, 1, 8187, 0, 0, 0, 51, 0, 0, 0, 519, 0, 6298, 6039, 0, 0, 14, 0, 1, 0, 0, 0, 0, 0, 0, 13, 0, 280, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 3700, 5, 0, 0, 0, 0, 0, 153, 0, 0, 0, 0, 4536, 0, 0, 0, 0, 10, 0, 0, 0, 0] payload_rows_per_dst=[16364, 3986, 14106, 16384, 12524, 16378, 15004, 14886] dup_rows_per_dst=[26, 126, 1330, 6270, 722, 586, 5004, 7376] combine_rows_per_src=[16364, 3986, 14106, 16384, 12524, 16378, 15004, 14886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104632 payload_rows_remote=89732 dup_rows=26440 dup_rows_remote=25094 dedup_ratio=0.201721 router_tokens_per_expert=[0, 0, 0, 1, 8, 3, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 436, 0, 0, 1234, 6896, 0, 0, 8190, 5, 409, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 6, 0, 2, 691, 0, 0, 1, 7422, 0, 1, 0, 0, 0, 0, 0, 0, 7, 3, 0, 2925, 0, 0, 0, 1, 8186, 0, 0, 0, 42, 0, 0, 0, 380, 0, 6664, 4832, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 484, 0, 0, 0, 0, 4, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3084, 0, 0, 4, 0, 0, 0, 492, 0, 1, 0, 0, 4903, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16382, 14900, 5860, 12446, 16376, 11954, 10338] dup_rows_per_dst=[22, 17968, 1346, 14, 2972, 984, 2678, 456] combine_rows_per_src=[16376, 16382, 14900, 5860, 12446, 16376, 11954, 10338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110094 payload_rows_remote=95026 dup_rows=20978 dup_rows_remote=15328 dedup_ratio=0.160049 router_tokens_per_expert=[0, 0, 0, 0, 2, 3, 1, 0, 0, 8176, 0, 0, 1, 0, 0, 0, 2, 4, 1, 0, 401, 0, 0, 995, 6440, 0, 0, 8190, 19, 544, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 844, 0, 0, 0, 7466, 0, 0, 1, 1, 0, 0, 0, 1, 0, 0, 0, 2621, 1, 0, 0, 0, 8190, 1, 0, 0, 39, 0, 0, 0, 520, 0, 6100, 5881, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 289, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4839, 3, 0, 0, 0, 0, 1, 455, 0, 0, 0, 0, 3462, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16354, 3822, 14976, 15484, 12602, 16382, 15406, 15068] dup_rows_per_dst=[12, 110, 1650, 4924, 1544, 608, 6480, 5650] combine_rows_per_src=[16354, 3822, 14976, 15484, 12602, 16382, 15406, 15068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=106172 payload_rows_remote=90466 dup_rows=24900 dup_rows_remote=18366 dedup_ratio=0.189972 router_tokens_per_expert=[0, 0, 0, 0, 7, 5, 0, 0, 0, 8155, 0, 0, 0, 0, 1, 0, 0, 2, 0, 1, 435, 0, 0, 1327, 6408, 0, 0, 8190, 57, 553, 0, 0, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 1, 698, 0, 0, 9, 7047, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3721, 0, 0, 0, 0, 8175, 0, 0, 0, 37, 0, 2, 0, 286, 0, 6794, 5557, 0, 0, 13, 0, 0, 1, 0, 0, 0, 0, 0, 30, 0, 558, 0, 0, 0, 0, 23, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 2, 2982, 6, 0, 0, 0, 0, 0, 124, 0, 0, 0, 7, 4305, 1, 0, 0, 0, 3, 2, 0, 0, 0] payload_rows_per_dst=[16318, 6796, 15570, 15706, 11516, 16366, 15140, 8760] dup_rows_per_dst=[18, 2152, 10156, 6534, 274, 1208, 4434, 124] combine_rows_per_src=[16318, 6796, 15570, 15706, 11516, 16366, 15140, 8760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109034 payload_rows_remote=93898 dup_rows=22038 dup_rows_remote=17228 dedup_ratio=0.168137 router_tokens_per_expert=[0, 0, 0, 5, 5, 9, 1, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 423, 0, 0, 1171, 7385, 0, 0, 8191, 7, 597, 0, 3, 0, 0, 1, 0, 0, 2, 0, 0, 0, 0, 12, 299, 0, 0, 0, 6610, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2879, 0, 0, 0, 2, 8180, 0, 0, 0, 83, 0, 0, 0, 614, 0, 6385, 5211, 0, 0, 20, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 330, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3582, 0, 0, 2, 3, 0, 0, 608, 0, 0, 0, 1, 4725, 0, 0, 0, 0, 11, 0, 0, 0, 0] payload_rows_per_dst=[16322, 4256, 13362, 16382, 11346, 16368, 15136, 15862] dup_rows_per_dst=[40, 152, 486, 5764, 510, 678, 4810, 9598] combine_rows_per_src=[16322, 4256, 13362, 16382, 11346, 16368, 15136, 15862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109516 payload_rows_remote=93134 dup_rows=21556 dup_rows_remote=21128 dedup_ratio=0.164459 router_tokens_per_expert=[0, 0, 2, 0, 0, 4, 1, 0, 0, 8138, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 395, 0, 0, 1696, 6105, 1, 0, 8183, 21, 401, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 2, 0, 448, 0, 0, 0, 7162, 0, 4, 2, 0, 0, 0, 3, 0, 3, 4, 0, 3688, 0, 2, 1, 0, 8185, 0, 0, 1, 51, 0, 0, 0, 374, 2, 6355, 5299, 0, 2, 12, 0, 0, 1, 0, 3, 2, 0, 1, 25, 1, 177, 0, 0, 1, 0, 9, 0, 1, 13, 1, 0, 0, 0, 0, 0, 1, 0, 4428, 1, 0, 2, 2, 2, 0, 706, 0, 0, 0, 6, 3572, 0, 0, 0, 0, 27, 0, 0, 0, 0] payload_rows_per_dst=[16288, 4792, 14378, 16382, 11068, 16382, 15412, 14814] dup_rows_per_dst=[6, 242, 850, 7398, 414, 428, 6200, 6018] combine_rows_per_src=[16288, 4792, 14378, 16382, 11068, 16382, 15412, 14814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=795508736 dispatch_meta_bytes=471304 combine_payload_bytes=795508736 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=795980040 ingress_bytes=1022001152 link_bytes=1022001152 token_bytes=1591488776 weight_movement_bytes=226492416 total_nvlink_bytes=1817981192 weight_to_token_ratio=0.142315 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.040441 gemm_active_groups=15 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.296041318e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=735084544 dispatch_meta_bytes=459304 combine_payload_bytes=735084544 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=735543848 ingress_bytes=810582016 link_bytes=810582016 token_bytes=1470628392 weight_movement_bytes=75497472 total_nvlink_bytes=1546125864 weight_to_token_ratio=0.051337 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=55936 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.027932094e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=778452992 dispatch_meta_bytes=441416 combine_payload_bytes=778452992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=778894408 ingress_bytes=853950464 link_bytes=853950464 token_bytes=1557347400 weight_movement_bytes=75497472 total_nvlink_bytes=1632844872 weight_to_token_ratio=0.048478 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=35456 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082929391e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=741097472 dispatch_meta_bytes=435328 combine_payload_bytes=741097472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=741532800 ingress_bytes=816594944 link_bytes=816594944 token_bytes=1482630272 weight_movement_bytes=75497472 total_nvlink_bytes=1558127744 weight_to_token_ratio=0.050921 gemm_rows_defined=67328 gemm_rows_charged=69632 gemm_tail_pad_rows=2304 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.033088 gemm_active_groups=17 gemm_peak_group_rows=39552 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.035557333e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97042 payload_rows_remote=80672 dup_rows=34030 dup_rows_remote=33850 dedup_ratio=0.259628 router_tokens_per_expert=[0, 0, 0, 0, 1, 13, 3, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 396, 0, 0, 1598, 6592, 2, 0, 8190, 11, 934, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 266, 0, 0, 1, 6851, 1, 0, 0, 0, 0, 0, 0, 0, 2, 3, 0, 3416, 0, 0, 0, 0, 8182, 0, 0, 0, 109, 0, 0, 2, 400, 0, 6212, 4784, 2, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 185, 0, 0, 0, 2, 10, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 3164, 3, 0, 1, 1, 0, 0, 920, 0, 0, 0, 0, 5034, 0, 0, 1, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16352, 13750, 6840, 15914, 10540, 6340, 10936] dup_rows_per_dst=[180, 18926, 500, 4, 13210, 220, 2, 988] combine_rows_per_src=[16370, 16352, 13750, 6840, 15914, 10540, 6340, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=769212416 dispatch_meta_bytes=444504 combine_payload_bytes=769212416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=769656920 ingress_bytes=844709888 link_bytes=844709888 token_bytes=1538869336 weight_movement_bytes=75497472 total_nvlink_bytes=1614366808 weight_to_token_ratio=0.049060 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=36224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.071211040e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=762953728 dispatch_meta_bytes=457048 combine_payload_bytes=762953728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=763410776 ingress_bytes=838451200 link_bytes=838451200 token_bytes=1526364504 weight_movement_bytes=75497472 total_nvlink_bytes=1601861976 weight_to_token_ratio=0.049462 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.036765 gemm_active_groups=15 gemm_peak_group_rows=62464 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.063274143e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101436 payload_rows_remote=85052 dup_rows=29636 dup_rows_remote=11010 dedup_ratio=0.226105 router_tokens_per_expert=[4, 1, 0, 2, 3, 14, 2, 0, 0, 8161, 0, 0, 0, 0, 0, 1, 0, 3, 0, 1, 377, 1, 2, 1231, 6902, 0, 0, 8190, 4, 787, 0, 7, 0, 0, 0, 1, 0, 0, 0, 4, 0, 0, 2, 473, 0, 0, 2, 6374, 0, 0, 2, 0, 0, 1, 3, 0, 0, 0, 0, 3554, 0, 0, 0, 3, 8187, 0, 0, 0, 86, 0, 0, 0, 406, 1, 5637, 5843, 0, 0, 32, 0, 0, 2, 0, 0, 1, 0, 0, 15, 0, 576, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 7, 3524, 0, 0, 0, 1, 0, 0, 260, 0, 0, 0, 1, 4819, 0, 0, 0, 2, 17, 0, 0, 0, 0] payload_rows_per_dst=[16336, 16384, 12900, 7120, 15244, 16380, 7068, 10004] dup_rows_per_dst=[40, 18626, 812, 6, 8766, 1188, 4, 194] combine_rows_per_src=[16336, 16384, 12900, 7120, 15244, 16380, 7068, 10004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130408 payload_rows_remote=114078 dup_rows=664 dup_rows_remote=662 dedup_ratio=0.005066 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 330, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7888, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16330, 16384, 15778, 16384] dup_rows_per_dst=[0, 0, 660, 0, 2, 0, 0, 2] combine_rows_per_src=[16382, 16384, 16384, 16382, 16330, 16384, 15778, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=660865024 dispatch_meta_bytes=458088 combine_payload_bytes=660865024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=661323112 ingress_bytes=736362496 link_bytes=736362496 token_bytes=1322188136 weight_movement_bytes=75497472 total_nvlink_bytes=1397685608 weight_to_token_ratio=0.057100 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.338112958e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129774 payload_rows_remote=113390 dup_rows=1298 dup_rows_remote=12 dedup_ratio=0.009903 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 643, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7549, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16372, 16384, 15098, 16384] dup_rows_per_dst=[6, 0, 1286, 0, 0, 0, 0, 6] combine_rows_per_src=[16384, 16384, 16384, 16384, 16372, 16384, 15098, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124632 payload_rows_remote=108248 dup_rows=6440 dup_rows_remote=6438 dedup_ratio=0.049133 router_tokens_per_expert=[8192, 0, 1, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 394, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 7795, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16276, 16380, 10768, 16308, 16380, 16382, 15754, 16384] dup_rows_per_dst=[10, 0, 626, 108, 296, 2, 5396, 2] combine_rows_per_src=[16276, 16380, 10768, 16308, 16380, 16382, 15754, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130436 payload_rows_remote=114054 dup_rows=636 dup_rows_remote=636 dedup_ratio=0.004852 router_tokens_per_expert=[8192, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 310, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7885, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16366, 16384, 15770, 16382] dup_rows_per_dst=[12, 0, 622, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16382, 16366, 16384, 15770, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=696745984 dispatch_meta_bytes=384248 combine_payload_bytes=696745984 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=697130232 ingress_bytes=1074233344 link_bytes=1074233344 token_bytes=1393876216 weight_movement_bytes=377487360 total_nvlink_bytes=1771363576 weight_to_token_ratio=0.270818 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=25344 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.362279090e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=934526976 dispatch_meta_bytes=458960 combine_payload_bytes=934526976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=934985936 ingress_bytes=1010024448 link_bytes=1010024448 token_bytes=1869512912 weight_movement_bytes=75497472 total_nvlink_bytes=1945010384 weight_to_token_ratio=0.040383 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.280853172e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130478 payload_rows_remote=114680 dup_rows=594 dup_rows_remote=594 dedup_ratio=0.004532 router_tokens_per_expert=[8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16380, 16384, 15798, 16384] dup_rows_per_dst=[4, 0, 576, 0, 0, 0, 0, 14] combine_rows_per_src=[16384, 16380, 16384, 16384, 16380, 16384, 15798, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=928890880 dispatch_meta_bytes=453608 combine_payload_bytes=928890880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=929344488 ingress_bytes=1004388352 link_bytes=1004388352 token_bytes=1858235368 weight_movement_bytes=75497472 total_nvlink_bytes=1933732840 weight_to_token_ratio=0.040629 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.273705809e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130334 payload_rows_remote=113954 dup_rows=738 dup_rows_remote=732 dedup_ratio=0.005630 router_tokens_per_expert=[8185, 1, 0, 1, 0, 0, 4, 1, 0, 0, 4, 0, 0, 2, 0, 0, 0, 0, 5, 1, 8187, 8184, 1, 8183, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 344, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8187, 8187, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8170, 2, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7852, 3, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16378, 16378, 16348, 16380, 15712, 16376] dup_rows_per_dst=[12, 6, 694, 6, 6, 6, 0, 8] combine_rows_per_src=[16384, 16378, 16378, 16378, 16348, 16380, 15712, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=886767616 dispatch_meta_bytes=458744 combine_payload_bytes=886767616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=887226360 ingress_bytes=962265088 link_bytes=962265088 token_bytes=1773993976 weight_movement_bytes=75497472 total_nvlink_bytes=1849491448 weight_to_token_ratio=0.042558 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.220287581e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=934330368 dispatch_meta_bytes=458760 combine_payload_bytes=934330368 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=934789128 ingress_bytes=1236320256 link_bytes=1236320256 token_bytes=1869119496 weight_movement_bytes=301989888 total_nvlink_bytes=2171109384 weight_to_token_ratio=0.161568 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=62592 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.567828110e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=939458560 dispatch_meta_bytes=461096 combine_payload_bytes=939458560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939919656 ingress_bytes=1014956032 link_bytes=1014956032 token_bytes=1879378216 weight_movement_bytes=75497472 total_nvlink_bytes=1954875688 weight_to_token_ratio=0.040172 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=62848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287107114e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=933511168 dispatch_meta_bytes=458744 combine_payload_bytes=933511168 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933969912 ingress_bytes=1009008640 link_bytes=1009008640 token_bytes=1867481080 weight_movement_bytes=75497472 total_nvlink_bytes=1942978552 weight_to_token_ratio=0.040427 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.279564984e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130632 payload_rows_remote=114248 dup_rows=440 dup_rows_remote=402 dedup_ratio=0.003357 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 195, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8011, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16336, 16378, 16022, 16364] dup_rows_per_dst=[38, 12, 390, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16336, 16378, 16022, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131054 payload_rows_remote=114670 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000137 router_tokens_per_expert=[0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 8183, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16372, 16384, 16380, 16384, 16382] dup_rows_per_dst=[2, 0, 2, 0, 0, 4, 2, 8] combine_rows_per_src=[16384, 16384, 16384, 16372, 16384, 16380, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130874 payload_rows_remote=114490 dup_rows=198 dup_rows_remote=198 dedup_ratio=0.001511 router_tokens_per_expert=[0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8093, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16364, 16384, 16384, 16384, 16384, 16384, 16384, 16206] dup_rows_per_dst=[0, 198, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16364, 16384, 16384, 16384, 16384, 16384, 16384, 16206] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130274 payload_rows_remote=113890 dup_rows=798 dup_rows_remote=798 dedup_ratio=0.006088 router_tokens_per_expert=[8191, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8190, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 384, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7872, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16356, 16256, 16384, 15744, 16382] dup_rows_per_dst=[28, 0, 768, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16356, 16256, 16384, 15744, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129092 payload_rows_remote=112722 dup_rows=1980 dup_rows_remote=68 dedup_ratio=0.015106 router_tokens_per_expert=[0, 0, 8174, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 8171, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16350, 14516, 16378, 16376, 16380, 16384, 16338, 16370] dup_rows_per_dst=[0, 48, 0, 6, 2, 8, 4, 1912] combine_rows_per_src=[16350, 14516, 16378, 16376, 16380, 16384, 16338, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=935919616 dispatch_meta_bytes=458600 combine_payload_bytes=935919616 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=936378216 ingress_bytes=1086914560 link_bytes=1086914560 token_bytes=1872297832 weight_movement_bytes=150994944 total_nvlink_bytes=2023292776 weight_to_token_ratio=0.080647 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378360657e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130806 payload_rows_remote=114422 dup_rows=266 dup_rows_remote=264 dedup_ratio=0.002029 router_tokens_per_expert=[0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8072, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16360, 16384, 16380, 16384, 16384, 16384, 16384, 16146] dup_rows_per_dst=[0, 260, 0, 2, 0, 2, 2, 0] combine_rows_per_src=[16360, 16384, 16380, 16384, 16384, 16384, 16384, 16146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 2, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=939376640 dispatch_meta_bytes=458752 combine_payload_bytes=939376640 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939835392 ingress_bytes=1090371584 link_bytes=1090371584 token_bytes=1879212032 weight_movement_bytes=150994944 total_nvlink_bytes=2030206976 weight_to_token_ratio=0.080350 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382744650e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130674 payload_rows_remote=114290 dup_rows=398 dup_rows_remote=396 dedup_ratio=0.003036 router_tokens_per_expert=[0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7996, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16378, 16384, 16384, 16384, 16384, 16384, 16384, 15992] dup_rows_per_dst=[0, 396, 0, 0, 0, 0, 2, 0] combine_rows_per_src=[16378, 16384, 16384, 16384, 16384, 16384, 16384, 15992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=937902080 dispatch_meta_bytes=458752 combine_payload_bytes=937902080 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=938360832 ingress_bytes=1164394496 link_bytes=1164394496 token_bytes=1876262912 weight_movement_bytes=226492416 total_nvlink_bytes=2102755328 weight_to_token_ratio=0.120715 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.476616122e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=923418624 dispatch_meta_bytes=451160 combine_payload_bytes=923418624 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=923869784 ingress_bytes=1074413568 link_bytes=1074413568 token_bytes=1847288408 weight_movement_bytes=150994944 total_nvlink_bytes=1998283352 weight_to_token_ratio=0.081739 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.362507639e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=932986880 dispatch_meta_bytes=458752 combine_payload_bytes=932986880 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=933445632 ingress_bytes=1159479296 link_bytes=1159479296 token_bytes=1866432512 weight_movement_bytes=226492416 total_nvlink_bytes=2092924928 weight_to_token_ratio=0.121350 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.470382956e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130968 payload_rows_remote=114604 dup_rows=104 dup_rows_remote=104 dedup_ratio=0.000793 router_tokens_per_expert=[0, 4, 8184, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 25, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 5, 8181, 0, 3, 7, 0, 0, 0, 0, 0, 8179, 0, 15, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 0, 3, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 7, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7930, 0, 236, 0, 5, 0, 0, 0, 0, 0, 0, 8184, 0, 0] payload_rows_per_dst=[16384, 16372, 16366, 16372, 16384, 16364, 16384, 16342] dup_rows_per_dst=[8, 0, 12, 46, 22, 0, 16, 0] combine_rows_per_src=[16384, 16372, 16366, 16372, 16384, 16364, 16384, 16342] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=937345024 dispatch_meta_bytes=458744 combine_payload_bytes=937345024 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=937803768 ingress_bytes=1239334912 link_bytes=1239334912 token_bytes=1875148792 weight_movement_bytes=301989888 total_nvlink_bytes=2177138680 weight_to_token_ratio=0.161048 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.571651118e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=936263680 dispatch_meta_bytes=458744 combine_payload_bytes=936263680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=936722424 ingress_bytes=1011761152 link_bytes=1011761152 token_bytes=1872986104 weight_movement_bytes=75497472 total_nvlink_bytes=1948483576 weight_to_token_ratio=0.040309 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283055557e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=938835968 dispatch_meta_bytes=458832 combine_payload_bytes=938835968 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939294800 ingress_bytes=1089830912 link_bytes=1089830912 token_bytes=1878130768 weight_movement_bytes=150994944 total_nvlink_bytes=2029125712 weight_to_token_ratio=0.080396 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382059002e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114676 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16380, 16384, 16374, 16384] dup_rows_per_dst=[0, 8, 6, 2, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16380, 16384, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130764 payload_rows_remote=114406 dup_rows=308 dup_rows_remote=196 dedup_ratio=0.002350 router_tokens_per_expert=[0, 0, 8141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8071, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16358, 16346, 16384, 16368, 16382, 16380, 16384, 16162] dup_rows_per_dst=[112, 174, 16, 0, 0, 0, 0, 6] combine_rows_per_src=[16358, 16346, 16384, 16368, 16382, 16380, 16384, 16162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131050 payload_rows_remote=114666 dup_rows=22 dup_rows_remote=22 dedup_ratio=0.000168 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8192, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16364, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 18, 0, 0, 0, 4] combine_rows_per_src=[16384, 16382, 16384, 16384, 16364, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130700 payload_rows_remote=114316 dup_rows=372 dup_rows_remote=184 dedup_ratio=0.002838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16376, 16208, 16380, 16384, 16200, 16384, 16384, 16384] dup_rows_per_dst=[4, 8, 0, 2, 0, 0, 170, 188] combine_rows_per_src=[16376, 16208, 16380, 16384, 16200, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114674 dup_rows=14 dup_rows_remote=12 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16372, 16384, 16384, 16384] dup_rows_per_dst=[4, 6, 0, 2, 0, 0, 2, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 16372, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130194 payload_rows_remote=113810 dup_rows=878 dup_rows_remote=36 dedup_ratio=0.006699 router_tokens_per_expert=[0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 421, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16372, 16384, 16382, 16376, 16366, 16384, 16384, 15546] dup_rows_per_dst=[0, 842, 0, 2, 0, 14, 20, 0] combine_rows_per_src=[16372, 16384, 16382, 16376, 16366, 16384, 16384, 15546] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458768 combine_payload_bytes=939425792 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939884560 ingress_bytes=1165918208 link_bytes=1165918208 token_bytes=1879310352 weight_movement_bytes=226492416 total_nvlink_bytes=2105802768 weight_to_token_ratio=0.120519 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478548403e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=937213952 dispatch_meta_bytes=458408 combine_payload_bytes=937213952 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937672360 ingress_bytes=1012711424 link_bytes=1012711424 token_bytes=1874886312 weight_movement_bytes=75497472 total_nvlink_bytes=1950383784 weight_to_token_ratio=0.040268 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284260636e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16384] dup_rows_per_dst=[0, 6, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16376, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939343872 dispatch_meta_bytes=458752 combine_payload_bytes=939343872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939802624 ingress_bytes=1014841344 link_bytes=1014841344 token_bytes=1879146496 weight_movement_bytes=75497472 total_nvlink_bytes=1954643968 weight_to_token_ratio=0.040176 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286961674e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=936476672 dispatch_meta_bytes=458000 combine_payload_bytes=936476672 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=936934672 ingress_bytes=1313964032 link_bytes=1313964032 token_bytes=1873411344 weight_movement_bytes=377487360 total_nvlink_bytes=2250898704 weight_to_token_ratio=0.201497 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.666291347e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130686 payload_rows_remote=114310 dup_rows=386 dup_rows_remote=378 dedup_ratio=0.002945 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 4, 3, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 4, 0, 0, 0, 0, 0, 0, 0, 8, 8, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 9, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 8087, 0, 0, 4, 0, 0, 0, 0, 33, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 2, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 59, 0, 0, 0, 191, 8177, 8150, 0, 0, 0, 0, 1, 8182, 8174, 0, 0, 8] payload_rows_per_dst=[16370, 16384, 16366, 16368, 16248, 16376, 16194, 16380] dup_rows_per_dst=[6, 8, 16, 14, 0, 8, 14, 320] combine_rows_per_src=[16370, 16384, 16366, 16368, 16248, 16376, 16194, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939409408 dispatch_meta_bytes=458744 combine_payload_bytes=939409408 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=939868152 ingress_bytes=1014906880 link_bytes=1014906880 token_bytes=1879277560 weight_movement_bytes=75497472 total_nvlink_bytes=1954775032 weight_to_token_ratio=0.040174 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287044783e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=932331520 dispatch_meta_bytes=455384 combine_payload_bytes=932331520 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=932786904 ingress_bytes=1158823936 link_bytes=1158823936 token_bytes=1865118424 weight_movement_bytes=226492416 total_nvlink_bytes=2091610840 weight_to_token_ratio=0.121436 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=64512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.469551868e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939917312 ingress_bytes=1090453504 link_bytes=1090453504 token_bytes=1879375872 weight_movement_bytes=150994944 total_nvlink_bytes=2030370816 weight_to_token_ratio=0.080343 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382848536e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=936427520 dispatch_meta_bytes=458752 combine_payload_bytes=936427520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=936886272 ingress_bytes=1011924992 link_bytes=1011924992 token_bytes=1873313792 weight_movement_bytes=75497472 total_nvlink_bytes=1948811264 weight_to_token_ratio=0.040302 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283263329e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126686 payload_rows_remote=110302 dup_rows=4386 dup_rows_remote=4304 dedup_ratio=0.033463 router_tokens_per_expert=[0, 0, 13, 1, 0, 8168, 0, 0, 0, 3, 0, 0, 0, 8184, 8191, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6024, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 19, 8182, 0, 30, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2142, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16356, 16382, 12050, 16382, 16384, 16370, 16382, 16380] dup_rows_per_dst=[14, 0, 2, 0, 82, 2, 2, 4284] combine_rows_per_src=[16356, 16382, 12050, 16382, 16384, 16370, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130538 payload_rows_remote=114154 dup_rows=534 dup_rows_remote=534 dedup_ratio=0.004074 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16358, 16384, 16384, 16384, 15876] dup_rows_per_dst=[0, 32, 0, 0, 502, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16358, 16384, 16384, 16384, 15876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124646 payload_rows_remote=108262 dup_rows=6426 dup_rows_remote=2 dedup_ratio=0.049026 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3212, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4983, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16384, 16382, 16384, 9966] dup_rows_per_dst=[0, 0, 6424, 0, 2, 0, 0, 0] combine_rows_per_src=[16384, 16378, 16384, 16384, 16384, 16382, 16384, 9966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119668 payload_rows_remote=103284 dup_rows=11404 dup_rows_remote=6006 dedup_ratio=0.087006 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 8187, 8192, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5501, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 2697, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[10382, 16378, 11002, 16380, 16384, 16374, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 5946, 0, 56, 2, 5398] combine_rows_per_src=[10382, 16378, 11002, 16380, 16384, 16374, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115982 payload_rows_remote=105336 dup_rows=15090 dup_rows_remote=15090 dedup_ratio=0.115128 router_tokens_per_expert=[0, 0, 82, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6866, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16350, 10646, 16384, 16382, 16384, 7072] dup_rows_per_dst=[164, 0, 13566, 0, 4, 0, 0, 1356] combine_rows_per_src=[16384, 16380, 16350, 10646, 16384, 16382, 16384, 7072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130932 payload_rows_remote=114548 dup_rows=140 dup_rows_remote=28 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 69, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16246, 16384, 16382, 16384] dup_rows_per_dst=[10, 112, 0, 2, 0, 0, 16, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16246, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=903593984 dispatch_meta_bytes=458424 combine_payload_bytes=903593984 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=904052408 ingress_bytes=1054588928 link_bytes=1054588928 token_bytes=1807646392 weight_movement_bytes=150994944 total_nvlink_bytes=1958641336 weight_to_token_ratio=0.083531 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.337367205e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122694 payload_rows_remote=106310 dup_rows=8378 dup_rows_remote=8378 dedup_ratio=0.063919 router_tokens_per_expert=[0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4016, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4188, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16370, 8032, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 8378] combine_rows_per_src=[16374, 16370, 8032, 16382, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=935149568 dispatch_meta_bytes=458752 combine_payload_bytes=935149568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935608320 ingress_bytes=1010647040 link_bytes=1010647040 token_bytes=1870757888 weight_movement_bytes=75497472 total_nvlink_bytes=1946255360 weight_to_token_ratio=0.040357 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281642706e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=886882304 dispatch_meta_bytes=433056 combine_payload_bytes=886882304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=887315360 ingress_bytes=962379776 link_bytes=962379776 token_bytes=1774197664 weight_movement_bytes=75497472 total_nvlink_bytes=1849695136 weight_to_token_ratio=0.042553 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=32896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.220433021e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117140 payload_rows_remote=100772 dup_rows=13932 dup_rows_remote=13916 dedup_ratio=0.106293 router_tokens_per_expert=[0, 0, 2899, 0, 0, 7735, 0, 0, 6, 0, 0, 0, 0, 8179, 8180, 0, 0, 7971, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 13, 10, 0, 0, 0, 0, 1529, 0, 0, 0, 0, 0, 0, 0, 26, 0, 8066, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 1, 0, 110, 0, 9, 0, 0, 0, 0, 0, 0, 0, 9, 0, 5, 3778, 0, 7, 0, 0, 0, 0, 0, 0, 6] payload_rows_per_dst=[16378, 15962, 3132, 16184, 16362, 16368, 16380, 16374] dup_rows_per_dst=[4902, 0, 22, 0, 1172, 16, 236, 7584] combine_rows_per_src=[16378, 15962, 3132, 16184, 16362, 16368, 16380, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=846102528 dispatch_meta_bytes=437160 combine_payload_bytes=846102528 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=846539688 ingress_bytes=921600000 link_bytes=921600000 token_bytes=1692642216 weight_movement_bytes=75497472 total_nvlink_bytes=1768139688 weight_to_token_ratio=0.044603 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=35712 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.168718525e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=862912512 dispatch_meta_bytes=481704 combine_payload_bytes=862912512 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=863394216 ingress_bytes=1089404928 link_bytes=1089404928 token_bytes=1726306728 weight_movement_bytes=226492416 total_nvlink_bytes=1952799144 weight_to_token_ratio=0.131201 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381518794e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=938377216 dispatch_meta_bytes=458304 combine_payload_bytes=938377216 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938835520 ingress_bytes=1089372160 link_bytes=1089372160 token_bytes=1877212736 weight_movement_bytes=150994944 total_nvlink_bytes=2028207680 weight_to_token_ratio=0.080436 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381477240e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=870891520 dispatch_meta_bytes=458752 combine_payload_bytes=870891520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=871350272 ingress_bytes=946388992 link_bytes=946388992 token_bytes=1742241792 weight_movement_bytes=75497472 total_nvlink_bytes=1817739264 weight_to_token_ratio=0.043334 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.200154456e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=825524224 dispatch_meta_bytes=458752 combine_payload_bytes=825524224 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=825982976 ingress_bytes=976519168 link_bytes=976519168 token_bytes=1651507200 weight_movement_bytes=150994944 total_nvlink_bytes=1802502144 weight_to_token_ratio=0.091429 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.238363760e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130914 payload_rows_remote=114582 dup_rows=158 dup_rows_remote=158 dedup_ratio=0.001205 router_tokens_per_expert=[0, 0, 23, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8114, 8190, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 78, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8188, 8166, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16280, 16384, 16384, 16384, 16332, 16384, 16384, 16382] dup_rows_per_dst=[0, 4, 150, 4, 0, 0, 0, 0] combine_rows_per_src=[16280, 16384, 16384, 16384, 16332, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131034 payload_rows_remote=114650 dup_rows=38 dup_rows_remote=0 dedup_ratio=0.000290 router_tokens_per_expert=[0, 0, 11, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 8150, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16346, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 38, 0, 0, 0, 0, 0] combine_rows_per_src=[16346, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122724 payload_rows_remote=106340 dup_rows=8348 dup_rows_remote=7394 dedup_ratio=0.063690 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8186, 8192, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4757, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16358, 16384, 16384, 15116, 16372, 16210, 9516] dup_rows_per_dst=[954, 488, 6900, 0, 0, 0, 0, 6] combine_rows_per_src=[16384, 16358, 16384, 16384, 15116, 16372, 16210, 9516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126484 payload_rows_remote=110100 dup_rows=4588 dup_rows_remote=2916 dedup_ratio=0.035004 router_tokens_per_expert=[0, 0, 14, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 8165, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14704, 16370, 13560, 16348, 16384, 16350, 16384, 16384] dup_rows_per_dst=[0, 0, 8, 4, 2870, 0, 34, 1672] combine_rows_per_src=[14704, 16370, 13560, 16348, 16384, 16350, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 85, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8104, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126566 payload_rows_remote=110190 dup_rows=4506 dup_rows_remote=4506 dedup_ratio=0.034378 router_tokens_per_expert=[0, 0, 1, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 8183, 8161, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2246, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 5984, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16376, 16380, 16384, 16384, 16366, 16324, 11968] dup_rows_per_dst=[10, 0, 4444, 0, 30, 0, 0, 22] combine_rows_per_src=[16384, 16376, 16380, 16384, 16384, 16366, 16324, 11968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=938655744 dispatch_meta_bytes=458960 combine_payload_bytes=938655744 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=939114704 ingress_bytes=1316143104 link_bytes=1316143104 token_bytes=1877770448 weight_movement_bytes=377487360 total_nvlink_bytes=2255257808 weight_to_token_ratio=0.201030 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=64000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.669054717e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 295, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7896, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939212800 dispatch_meta_bytes=458600 combine_payload_bytes=939212800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939671400 ingress_bytes=1014710272 link_bytes=1014710272 token_bytes=1878884200 weight_movement_bytes=75497472 total_nvlink_bytes=1954381672 weight_to_token_ratio=0.040182 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286795456e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=871137280 dispatch_meta_bytes=454936 combine_payload_bytes=871137280 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=871592216 ingress_bytes=1248624640 link_bytes=1248624640 token_bytes=1742729496 weight_movement_bytes=377487360 total_nvlink_bytes=2120216856 weight_to_token_ratio=0.216607 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=62080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.583431801e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127300 payload_rows_remote=110916 dup_rows=3772 dup_rows_remote=3738 dedup_ratio=0.028778 router_tokens_per_expert=[0, 0, 449, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7936, 7311, 0, 2, 0, 0, 0, 10, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1333, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4, 8132, 7413, 0, 0, 7, 0, 8179, 0, 0, 0, 9, 8175, 0, 0, 0, 13, 8178, 0, 0, 0, 0, 0, 0, 0, 2, 0, 5, 0, 0, 12, 0, 12, 8, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 95, 0, 0, 0, 31, 0, 0, 0, 0, 6, 9, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15888, 16382, 16352, 16362, 14870, 16384, 16360, 14702] dup_rows_per_dst=[884, 12, 2606, 8, 14, 34, 202, 12] combine_rows_per_src=[15888, 16382, 16352, 16362, 14870, 16384, 16360, 14702] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=901939200 dispatch_meta_bytes=452064 combine_payload_bytes=901939200 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=902391264 ingress_bytes=1052934144 link_bytes=1052934144 token_bytes=1804330464 weight_movement_bytes=150994944 total_nvlink_bytes=1955325408 weight_to_token_ratio=0.083685 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.335268706e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458744 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939966456 ingress_bytes=1015005184 link_bytes=1015005184 token_bytes=1879474168 weight_movement_bytes=75497472 total_nvlink_bytes=1954971640 weight_to_token_ratio=0.040169 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287169446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=902676480 dispatch_meta_bytes=458784 combine_payload_bytes=902676480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=903135264 ingress_bytes=978173952 link_bytes=978173952 token_bytes=1805811744 weight_movement_bytes=75497472 total_nvlink_bytes=1881309216 weight_to_token_ratio=0.041808 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.240462260e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939966464 ingress_bytes=1090502656 link_bytes=1090502656 token_bytes=1879474176 weight_movement_bytes=150994944 total_nvlink_bytes=2030469120 weight_to_token_ratio=0.080339 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382910868e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=908623872 dispatch_meta_bytes=458616 combine_payload_bytes=908623872 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=909082488 ingress_bytes=984121344 link_bytes=984121344 token_bytes=1817706360 weight_movement_bytes=75497472 total_nvlink_bytes=1893203832 weight_to_token_ratio=0.041534 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.248004390e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129624 payload_rows_remote=113242 dup_rows=1448 dup_rows_remote=1448 dedup_ratio=0.011047 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2, 0, 0, 8164, 8191, 0, 0, 0, 0, 8191, 0, 0, 8188, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 512, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7513, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 216, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 8191] payload_rows_per_dst=[16332, 16380, 16362, 15026, 16382, 16382, 16376, 16384] dup_rows_per_dst=[2, 0, 1006, 4, 0, 432, 0, 4] combine_rows_per_src=[16332, 16380, 16362, 15026, 16382, 16382, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128346 payload_rows_remote=111962 dup_rows=2726 dup_rows_remote=2100 dedup_ratio=0.020798 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 320, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6829, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 16384, 13658, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 626, 0, 0, 2096, 0, 4] combine_rows_per_src=[16384, 16384, 16384, 13658, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129558 payload_rows_remote=113174 dup_rows=1514 dup_rows_remote=58 dedup_ratio=0.011551 router_tokens_per_expert=[0, 0, 8, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8148, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 29, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 15082, 16384, 16172, 16384] dup_rows_per_dst=[1456, 0, 58, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 15082, 16384, 16172, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111830 payload_rows_remote=95446 dup_rows=19242 dup_rows_remote=19218 dedup_ratio=0.146805 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8188, 2, 0, 0, 0, 8192, 0, 0, 8189, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 167, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 222, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[4, 13528, 16384, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 3160, 15618, 4, 436, 0, 24] combine_rows_per_src=[4, 13528, 16384, 16384, 16384, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130200 payload_rows_remote=114674 dup_rows=872 dup_rows_remote=872 dedup_ratio=0.006653 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 0, 0, 0, 8192, 0, 0, 8189, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 352, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7763, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16376, 16384, 15526, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 708, 0, 4, 160, 0, 0] combine_rows_per_src=[16384, 16376, 16384, 15526, 16384, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 97, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 8070, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 8189, 8184, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16384, 16380, 16384, 16368, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 38, 0, 0, 0, 0] combine_rows_per_src=[16364, 16384, 16380, 16384, 16368, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128038 payload_rows_remote=111654 dup_rows=3034 dup_rows_remote=3034 dedup_ratio=0.023148 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1114, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16126, 16368, 16384, 13624, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2226, 250, 2, 556, 0, 0] combine_rows_per_src=[16126, 16368, 16384, 13624, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=927678464 dispatch_meta_bytes=458760 combine_payload_bytes=927678464 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928137224 ingress_bytes=1003175936 link_bytes=1003175936 token_bytes=1855815688 weight_movement_bytes=75497472 total_nvlink_bytes=1931313160 weight_to_token_ratio=0.040682 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.272168295e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126030 payload_rows_remote=109650 dup_rows=5042 dup_rows_remote=3074 dedup_ratio=0.038467 router_tokens_per_expert=[1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 5, 0, 7473, 8178, 0, 0, 0, 0, 8175, 5, 0, 8180, 0, 8173, 0, 7, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 6407, 0, 5, 0, 0, 0, 12, 10, 13, 5, 0, 8176, 0, 0, 0, 0, 0, 1, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 924, 63, 0, 0, 9, 0, 0, 0, 0, 8, 9, 0, 0, 0, 0, 1492, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 8172] payload_rows_per_dst=[14956, 16380, 12830, 16384, 16364, 16380, 16384, 16352] dup_rows_per_dst=[6, 12, 0, 60, 2, 1968, 2994, 0] combine_rows_per_src=[14956, 16380, 12830, 16384, 16364, 16380, 16384, 16352] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=917192704 dispatch_meta_bytes=456248 combine_payload_bytes=917192704 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=917648952 ingress_bytes=1068187648 link_bytes=1068187648 token_bytes=1834841656 weight_movement_bytes=150994944 total_nvlink_bytes=1985836600 weight_to_token_ratio=0.082293 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.354612296e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=927121408 dispatch_meta_bytes=452928 combine_payload_bytes=927121408 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=927574336 ingress_bytes=1153613824 link_bytes=1153613824 token_bytes=1854695744 weight_movement_bytes=226492416 total_nvlink_bytes=2081188160 weight_to_token_ratio=0.122118 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.462944712e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=939409408 dispatch_meta_bytes=462184 combine_payload_bytes=939409408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939871592 ingress_bytes=1014906880 link_bytes=1014906880 token_bytes=1879281000 weight_movement_bytes=75497472 total_nvlink_bytes=1954778472 weight_to_token_ratio=0.040174 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=57216 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287044783e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=781893632 dispatch_meta_bytes=458656 combine_payload_bytes=781893632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=782352288 ingress_bytes=857391104 link_bytes=857391104 token_bytes=1564245920 weight_movement_bytes=75497472 total_nvlink_bytes=1639743392 weight_to_token_ratio=0.048264 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.087292607e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458752 combine_payload_bytes=939196416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939655168 ingress_bytes=1014693888 link_bytes=1014693888 token_bytes=1878851584 weight_movement_bytes=75497472 total_nvlink_bytes=1954349056 weight_to_token_ratio=0.040183 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286774679e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=914669568 dispatch_meta_bytes=458752 combine_payload_bytes=914669568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=915128320 ingress_bytes=990167040 link_bytes=990167040 token_bytes=1829797888 weight_movement_bytes=75497472 total_nvlink_bytes=1905295360 weight_to_token_ratio=0.041260 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.255671183e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=898252800 dispatch_meta_bytes=450896 combine_payload_bytes=898252800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=898703696 ingress_bytes=973750272 link_bytes=973750272 token_bytes=1796956496 weight_movement_bytes=75497472 total_nvlink_bytes=1872453968 weight_to_token_ratio=0.042014 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=60288 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.234852411e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131002 payload_rows_remote=114618 dup_rows=70 dup_rows_remote=24 dedup_ratio=0.000534 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 11, 0, 0, 0, 3, 1, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16332, 16384, 16382, 16384, 16376, 16384, 16376] dup_rows_per_dst=[22, 2, 0, 0, 46, 0, 0, 0] combine_rows_per_src=[16384, 16332, 16384, 16382, 16384, 16376, 16384, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 2, 0, 0, 2, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110000 payload_rows_remote=93616 dup_rows=21072 dup_rows_remote=2962 dedup_ratio=0.160767 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 8187, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 535, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7119, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 551, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16384, 9798, 14238, 16384, 6978, 13450, 16384] dup_rows_per_dst=[18110, 2924, 30, 0, 8, 0, 0, 0] combine_rows_per_src=[16384, 16384, 9798, 14238, 16384, 6978, 13450, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114682 dup_rows=6 dup_rows_remote=6 dedup_ratio=0.000046 router_tokens_per_expert=[0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16382] dup_rows_per_dst=[0, 0, 0, 0, 6, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938950656 dispatch_meta_bytes=458568 combine_payload_bytes=938950656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939409224 ingress_bytes=1014448128 link_bytes=1014448128 token_bytes=1878359880 weight_movement_bytes=75497472 total_nvlink_bytes=1953857352 weight_to_token_ratio=0.040193 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286463021e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130476 payload_rows_remote=114092 dup_rows=596 dup_rows_remote=584 dedup_ratio=0.004547 router_tokens_per_expert=[0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16228, 16362, 16372, 16384, 15982, 16384, 16380, 16384] dup_rows_per_dst=[0, 0, 0, 0, 26, 498, 60, 12] combine_rows_per_src=[16228, 16362, 16372, 16384, 15982, 16384, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130984 payload_rows_remote=114602 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.000671 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16360, 16384, 16384, 16384, 16326, 16382, 16380] dup_rows_per_dst=[0, 0, 0, 0, 88, 0, 0, 0] combine_rows_per_src=[16384, 16360, 16384, 16384, 16384, 16326, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124900 payload_rows_remote=108564 dup_rows=6172 dup_rows_remote=6172 dedup_ratio=0.047089 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 0, 0, 8192, 0, 0, 8172, 0, 8168, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 18, 0, 0, 1184, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1887, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192] payload_rows_per_dst=[16384, 16336, 16384, 10300, 16384, 16384, 16344, 16384] dup_rows_per_dst=[0, 0, 2406, 0, 2, 3764, 0, 0] combine_rows_per_src=[16384, 16336, 16384, 10300, 16384, 16384, 16344, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130388 payload_rows_remote=114020 dup_rows=684 dup_rows_remote=676 dedup_ratio=0.005219 router_tokens_per_expert=[0, 8177, 0, 0, 0, 0, 0, 5, 8092, 0, 0, 0, 0, 0, 74, 85, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 1, 0, 0, 0, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 2, 0, 0, 11, 15, 0, 0, 377, 0, 0, 0, 14, 0, 0, 0, 6, 0, 8, 0, 0, 1, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 1, 8171, 0, 0, 0, 3, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 7, 0, 8167, 15, 0, 8174, 0] payload_rows_per_dst=[16358, 15790, 16374, 16384, 16376, 16368, 16370, 16368] dup_rows_per_dst=[154, 0, 22, 474, 2, 8, 0, 24] combine_rows_per_src=[16358, 15790, 16374, 16384, 16376, 16368, 16370, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441408 weight_movement_bytes=75497472 total_nvlink_bytes=1954938880 weight_to_token_ratio=0.040170 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458752 combine_payload_bytes=939474944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939933696 ingress_bytes=1014972416 link_bytes=1014972416 token_bytes=1879408640 weight_movement_bytes=75497472 total_nvlink_bytes=1954906112 weight_to_token_ratio=0.040171 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287127892e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=766902272 dispatch_meta_bytes=386312 combine_payload_bytes=766902272 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=767288584 ingress_bytes=993394688 link_bytes=993394688 token_bytes=1534190856 weight_movement_bytes=226492416 total_nvlink_bytes=1760683272 weight_to_token_ratio=0.147630 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=56576 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.259764295e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=934641664 dispatch_meta_bytes=458704 combine_payload_bytes=934641664 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=935100368 ingress_bytes=1085636608 link_bytes=1085636608 token_bytes=1869742032 weight_movement_bytes=150994944 total_nvlink_bytes=2020736976 weight_to_token_ratio=0.080757 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.376740034e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=889356288 dispatch_meta_bytes=458944 combine_payload_bytes=889356288 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=889815232 ingress_bytes=1191346176 link_bytes=1191346176 token_bytes=1779171520 weight_movement_bytes=301989888 total_nvlink_bytes=2081161408 weight_to_token_ratio=0.169736 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.510794646e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938819584 dispatch_meta_bytes=458760 combine_payload_bytes=938819584 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939278344 ingress_bytes=1089814528 link_bytes=1089814528 token_bytes=1878097928 weight_movement_bytes=150994944 total_nvlink_bytes=2029092872 weight_to_token_ratio=0.080398 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382038224e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=934051840 dispatch_meta_bytes=458784 combine_payload_bytes=934051840 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=934510624 ingress_bytes=1160544256 link_bytes=1160544256 token_bytes=1868562464 weight_movement_bytes=226492416 total_nvlink_bytes=2095054880 weight_to_token_ratio=0.121212 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.471733476e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126258 payload_rows_remote=109874 dup_rows=4814 dup_rows_remote=4814 dedup_ratio=0.036728 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2407, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5864, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16226, 11728] dup_rows_per_dst=[0, 0, 2, 0, 0, 4812, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16226, 11728] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=900087808 dispatch_meta_bytes=458752 combine_payload_bytes=900087808 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=900546560 ingress_bytes=975585280 link_bytes=975585280 token_bytes=1800634368 weight_movement_bytes=75497472 total_nvlink_bytes=1876131840 weight_to_token_ratio=0.041928 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.237179459e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129138 payload_rows_remote=112754 dup_rows=1934 dup_rows_remote=1934 dedup_ratio=0.014755 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 966, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7268, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16298, 14536] dup_rows_per_dst=[0, 0, 0, 0, 2, 1932, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16298, 14536] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127232 payload_rows_remote=110848 dup_rows=3840 dup_rows_remote=3840 dedup_ratio=0.029297 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1904, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6295, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16342, 12590] dup_rows_per_dst=[0, 0, 0, 0, 32, 3808, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16342, 12590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130196 payload_rows_remote=113812 dup_rows=876 dup_rows_remote=870 dedup_ratio=0.006683 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16302, 16384, 15912, 16384, 16068, 16384, 16378] dup_rows_per_dst=[6, 752, 0, 0, 118, 0, 0, 0] combine_rows_per_src=[16384, 16302, 16384, 15912, 16384, 16068, 16384, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119416 payload_rows_remote=103032 dup_rows=11656 dup_rows_remote=1016 dedup_ratio=0.088928 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5319, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15390, 16378, 16384, 16384, 16162, 5974, 16360, 16384] dup_rows_per_dst=[0, 0, 40, 0, 14, 220, 742, 10640] combine_rows_per_src=[15390, 16378, 16384, 16384, 16162, 5974, 16360, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130926 payload_rows_remote=114546 dup_rows=146 dup_rows_remote=122 dedup_ratio=0.001114 router_tokens_per_expert=[0, 8175, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 12, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8137, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16276, 16350, 16384] dup_rows_per_dst=[0, 24, 0, 0, 122, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16276, 16350, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120452 payload_rows_remote=104198 dup_rows=10620 dup_rows_remote=10620 dedup_ratio=0.081024 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7617, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 5902, 16254, 16378] dup_rows_per_dst=[0, 0, 0, 0, 4, 134, 0, 10482] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 5902, 16254, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128416 payload_rows_remote=112552 dup_rows=2656 dup_rows_remote=1226 dedup_ratio=0.020264 router_tokens_per_expert=[0, 8092, 0, 0, 0, 0, 7, 0, 8177, 0, 0, 61, 0, 0, 0, 8154, 0, 0, 0, 0, 30, 0, 0, 473, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 81, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 8, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8153, 11, 0, 7908, 0, 10, 1, 0, 0, 0, 0, 696, 2, 19, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 6, 0, 1, 15, 0, 0, 202, 0, 0, 7037, 12, 0, 0, 1, 0, 0] payload_rows_per_dst=[16306, 16384, 16342, 16358, 16324, 15864, 16330, 14508] dup_rows_per_dst=[14, 998, 0, 124, 50, 1430, 0, 40] combine_rows_per_src=[16306, 16384, 16342, 16358, 16324, 15864, 16330, 14508] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=923680768 dispatch_meta_bytes=458752 combine_payload_bytes=923680768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=924139520 ingress_bytes=999178240 link_bytes=999178240 token_bytes=1847820288 weight_movement_bytes=75497472 total_nvlink_bytes=1923317760 weight_to_token_ratio=0.040858 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.267098653e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=908066816 dispatch_meta_bytes=458752 combine_payload_bytes=908066816 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=908525568 ingress_bytes=983564288 link_bytes=983564288 token_bytes=1816592384 weight_movement_bytes=75497472 total_nvlink_bytes=1892089856 weight_to_token_ratio=0.041560 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.247297964e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=932347904 dispatch_meta_bytes=458728 combine_payload_bytes=932347904 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=932806632 ingress_bytes=1234337792 link_bytes=1234337792 token_bytes=1865154536 weight_movement_bytes=301989888 total_nvlink_bytes=2167144424 weight_to_token_ratio=0.161911 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.565314066e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=844038144 dispatch_meta_bytes=416192 combine_payload_bytes=844038144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=844454336 ingress_bytes=919535616 link_bytes=919535616 token_bytes=1688492480 weight_movement_bytes=75497472 total_nvlink_bytes=1763989952 weight_to_token_ratio=0.044713 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=51328 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.166100596e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938360832 dispatch_meta_bytes=458672 combine_payload_bytes=938360832 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938819504 ingress_bytes=1089355776 link_bytes=1089355776 token_bytes=1877180336 weight_movement_bytes=150994944 total_nvlink_bytes=2028175280 weight_to_token_ratio=0.080437 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381456462e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131032 payload_rows_remote=114648 dup_rows=40 dup_rows_remote=40 dedup_ratio=0.000305 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16360, 16384, 16370, 16382, 16384] dup_rows_per_dst=[0, 0, 38, 0, 0, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16384, 16360, 16384, 16370, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=853590016 dispatch_meta_bytes=459272 combine_payload_bytes=853590016 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=854049288 ingress_bytes=929087488 link_bytes=929087488 token_bytes=1707639304 weight_movement_bytes=75497472 total_nvlink_bytes=1783136776 weight_to_token_ratio=0.044212 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178213714e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=922025984 dispatch_meta_bytes=455112 combine_payload_bytes=922025984 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=922481096 ingress_bytes=1148518400 link_bytes=1148518400 token_bytes=1844507080 weight_movement_bytes=226492416 total_nvlink_bytes=2070999496 weight_to_token_ratio=0.122793 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=51328 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.456482997e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939196416 dispatch_meta_bytes=458752 combine_payload_bytes=939196416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939655168 ingress_bytes=1014693888 link_bytes=1014693888 token_bytes=1878851584 weight_movement_bytes=75497472 total_nvlink_bytes=1954349056 weight_to_token_ratio=0.040183 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286774679e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=0 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16380, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114684 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000092 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16376, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 0, 8] combine_rows_per_src=[16384, 16380, 16384, 16376, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124256 payload_rows_remote=107872 dup_rows=6816 dup_rows_remote=5758 dedup_ratio=0.052002 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2878, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5349, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15446, 16384, 16262, 16384, 16384, 16314, 10698] dup_rows_per_dst=[1058, 0, 0, 0, 2, 5756, 0, 0] combine_rows_per_src=[16384, 15446, 16384, 16262, 16384, 16384, 16314, 10698] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127548 payload_rows_remote=111212 dup_rows=3524 dup_rows_remote=3524 dedup_ratio=0.026886 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 6543, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16336, 16384, 16384, 16384, 16384, 16206, 13086] dup_rows_per_dst=[0, 0, 0, 24, 178, 3320, 0, 2] combine_rows_per_src=[16384, 16336, 16384, 16384, 16384, 16384, 16206, 13086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 2] combine_rows_per_src=[16384, 16382, 16384, 16382, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131010 payload_rows_remote=114626 dup_rows=62 dup_rows_remote=14 dedup_ratio=0.000473 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[16342, 16370, 16384, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 10, 48] combine_rows_per_src=[16342, 16370, 16384, 16378, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130896 payload_rows_remote=114544 dup_rows=176 dup_rows_remote=176 dedup_ratio=0.001343 router_tokens_per_expert=[0, 0, 0, 0, 11, 8161, 0, 0, 0, 0, 0, 0, 0, 12, 0, 8144, 35, 7, 0, 8176, 0, 0, 2, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 16, 0, 8173, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 79, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 8159, 4, 0, 0, 19, 0, 8, 6, 0, 0, 0, 16, 0, 0, 0, 8174, 0, 0, 0, 16, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16328, 16382, 16344, 16378, 16350, 16352, 16382, 16380] dup_rows_per_dst=[6, 82, 0, 10, 2, 0, 72, 4] combine_rows_per_src=[16328, 16382, 16344, 16378, 16350, 16352, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458736 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939950064 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441392 weight_movement_bytes=75497472 total_nvlink_bytes=1954938864 weight_to_token_ratio=0.040170 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=883687424 dispatch_meta_bytes=454520 combine_payload_bytes=883687424 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=884141944 ingress_bytes=1261174784 link_bytes=1261174784 token_bytes=1767829368 weight_movement_bytes=377487360 total_nvlink_bytes=2145316728 weight_to_token_ratio=0.213532 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.599347150e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458784 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939950112 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441440 weight_movement_bytes=75497472 total_nvlink_bytes=1954938912 weight_to_token_ratio=0.040170 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130878 payload_rows_remote=114494 dup_rows=194 dup_rows_remote=194 dedup_ratio=0.001480 router_tokens_per_expert=[0, 0, 0, 8100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 8192, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 90, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16200, 16384, 16382, 16384, 16384, 16384, 16376, 16384] dup_rows_per_dst=[0, 2, 180, 4, 0, 8, 0, 0] combine_rows_per_src=[16200, 16384, 16382, 16384, 16384, 16384, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=911048704 dispatch_meta_bytes=458944 combine_payload_bytes=911048704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=911507648 ingress_bytes=986546176 link_bytes=986546176 token_bytes=1822556352 weight_movement_bytes=75497472 total_nvlink_bytes=1898053824 weight_to_token_ratio=0.041424 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.251079418e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939016192 dispatch_meta_bytes=458560 combine_payload_bytes=939016192 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939474752 ingress_bytes=1090011136 link_bytes=1090011136 token_bytes=1878490944 weight_movement_bytes=150994944 total_nvlink_bytes=2029485888 weight_to_token_ratio=0.080381 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382287551e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939950080 ingress_bytes=1165983744 link_bytes=1165983744 token_bytes=1879441408 weight_movement_bytes=226492416 total_nvlink_bytes=2105933824 weight_to_token_ratio=0.120510 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478631512e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=938344448 dispatch_meta_bytes=458880 combine_payload_bytes=938344448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938803328 ingress_bytes=1013841920 link_bytes=1013841920 token_bytes=1877147776 weight_movement_bytes=75497472 total_nvlink_bytes=1952645248 weight_to_token_ratio=0.040219 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285694264e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=937934848 dispatch_meta_bytes=458752 combine_payload_bytes=937934848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938393600 ingress_bytes=1013432320 link_bytes=1013432320 token_bytes=1876328448 weight_movement_bytes=75497472 total_nvlink_bytes=1951825920 weight_to_token_ratio=0.040237 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285174833e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131028 payload_rows_remote=114644 dup_rows=44 dup_rows_remote=0 dedup_ratio=0.000336 router_tokens_per_expert=[0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16346, 16384, 16384, 16384, 16384, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 44, 0, 0, 0, 0, 0] combine_rows_per_src=[16346, 16384, 16384, 16384, 16384, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[16384, 16380, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130768 payload_rows_remote=114384 dup_rows=304 dup_rows_remote=212 dedup_ratio=0.002319 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16306, 16384, 16384, 16384, 16306, 16238] dup_rows_per_dst=[92, 0, 0, 134, 32, 46, 0, 0] combine_rows_per_src=[16384, 16382, 16306, 16384, 16384, 16384, 16306, 16238] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130920 payload_rows_remote=114594 dup_rows=152 dup_rows_remote=152 dedup_ratio=0.001160 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 33, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0] payload_rows_per_dst=[16384, 16326, 16384, 16290, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 66, 0, 0, 0, 0, 86] combine_rows_per_src=[16384, 16326, 16384, 16290, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939163648 dispatch_meta_bytes=458576 combine_payload_bytes=939163648 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939622224 ingress_bytes=1090158592 link_bytes=1090158592 token_bytes=1878785872 weight_movement_bytes=150994944 total_nvlink_bytes=2029780816 weight_to_token_ratio=0.080368 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=62592 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382474546e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124996 payload_rows_remote=108612 dup_rows=6076 dup_rows_remote=3492 dedup_ratio=0.046356 router_tokens_per_expert=[0, 9, 0, 5291, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 8151, 0, 0, 8072, 8169, 0, 0, 107, 0, 8173, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 1586, 0, 0, 0, 0, 0, 3, 11, 0, 0, 0, 11, 0, 0, 8163, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 14, 13, 0, 0, 10, 0, 0, 1, 0, 0, 0, 0, 8, 0, 0, 0, 1295, 1, 0, 19, 10, 0, 0, 0, 0, 8169, 0, 0, 5, 2, 1, 0, 0, 12, 0, 0, 0, 0, 12, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[10604, 16194, 16374, 16354, 16384, 16384, 16364, 16338] dup_rows_per_dst=[0, 0, 3376, 22, 34, 2584, 60, 0] combine_rows_per_src=[10604, 16194, 16374, 16354, 16384, 16384, 16364, 16338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128576 payload_rows_remote=112192 dup_rows=2496 dup_rows_remote=2450 dedup_ratio=0.019043 router_tokens_per_expert=[0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 13924, 16384, 16354, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 2420, 0, 0, 0, 30, 46] combine_rows_per_src=[16378, 13924, 16384, 16354, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=939950080 ingress_bytes=1090486272 link_bytes=1090486272 token_bytes=1879441408 weight_movement_bytes=150994944 total_nvlink_bytes=2030436352 weight_to_token_ratio=0.080340 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382890090e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=937033728 dispatch_meta_bytes=458384 combine_payload_bytes=937033728 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=937492112 ingress_bytes=1163526144 link_bytes=1163526144 token_bytes=1874525840 weight_movement_bytes=226492416 total_nvlink_bytes=2101018256 weight_to_token_ratio=0.120827 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.475514929e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=6 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16374, 16380, 16384, 16382, 16382, 16384] dup_rows_per_dst=[6, 0, 0, 0, 14, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16374, 16380, 16384, 16382, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=938754048 dispatch_meta_bytes=458984 combine_payload_bytes=938754048 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=939213032 ingress_bytes=1089748992 link_bytes=1089748992 token_bytes=1877967080 weight_movement_bytes=150994944 total_nvlink_bytes=2028962024 weight_to_token_ratio=0.080403 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381955116e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=889749504 dispatch_meta_bytes=448416 combine_payload_bytes=889749504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=890197920 ingress_bytes=965246976 link_bytes=965246976 token_bytes=1779947424 weight_movement_bytes=75497472 total_nvlink_bytes=1855444896 weight_to_token_ratio=0.042416 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.224069034e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=919076864 dispatch_meta_bytes=458568 combine_payload_bytes=919076864 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=919535432 ingress_bytes=1070071808 link_bytes=1070071808 token_bytes=1838612296 weight_movement_bytes=150994944 total_nvlink_bytes=1989607240 weight_to_token_ratio=0.082124 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.357001677e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939966464 ingress_bytes=1090502656 link_bytes=1090502656 token_bytes=1879474176 weight_movement_bytes=150994944 total_nvlink_bytes=2030469120 weight_to_token_ratio=0.080339 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382910868e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458696 combine_payload_bytes=939360256 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939818952 ingress_bytes=1014857728 link_bytes=1014857728 token_bytes=1879179208 weight_movement_bytes=75497472 total_nvlink_bytes=1954676680 weight_to_token_ratio=0.040176 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286982451e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114688 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16356, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[28, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16356, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=22074110088 prefetch_bytes=905969664 reduce_bytes=754974720 weight_movement_bytes=1660944384 egress_bytes=11039813768 ingress_bytes=12695240704 link_bytes=12695240704 total_nvlink_bytes=23735054472 weight_to_token_ratio=0.075244 prefetch_share_of_ingress=0.071363 dedup_ratio=0.021151 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=10 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=11430 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.341612684e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131056 payload_rows_remote=114672 dup_rows=16 dup_rows_remote=16 dedup_ratio=0.000122 router_tokens_per_expert=[0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8184, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16370, 16384, 16384, 16382, 16384, 16384] dup_rows_per_dst=[16, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16370, 16384, 16384, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122350 payload_rows_remote=105966 dup_rows=8722 dup_rows_remote=2712 dedup_ratio=0.066544 router_tokens_per_expert=[0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 10374, 16384, 16350, 13792, 16298, 16384] dup_rows_per_dst=[6010, 2678, 0, 34, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 10374, 16384, 16350, 13792, 16298, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131046 payload_rows_remote=114688 dup_rows=26 dup_rows_remote=26 dedup_ratio=0.000198 router_tokens_per_expert=[0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8179, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16358, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 26, 0] combine_rows_per_src=[16384, 16358, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458864 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939982960 ingress_bytes=1241513984 link_bytes=1241513984 token_bytes=1879507056 weight_movement_bytes=301989888 total_nvlink_bytes=2181496944 weight_to_token_ratio=0.160675 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.574414488e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130120 payload_rows_remote=114528 dup_rows=952 dup_rows_remote=952 dedup_ratio=0.007263 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 14, 0, 57, 0, 0, 0, 8170, 0, 0, 3, 0, 0, 15, 0, 0, 11, 0, 11, 0, 14, 0, 0, 8154, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 2, 10, 0, 8118, 0, 0, 0, 0, 11, 0, 0, 393, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 4, 0, 9, 12, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 0, 7784, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16384, 16362, 16356, 16348, 15592, 16368, 16334] dup_rows_per_dst=[18, 64, 4, 800, 28, 0, 38, 0] combine_rows_per_src=[16376, 16384, 16362, 16356, 16348, 15592, 16368, 16334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[10, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16374, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458752 combine_payload_bytes=939393024 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939851776 ingress_bytes=1090387968 link_bytes=1090387968 token_bytes=1879244800 weight_movement_bytes=150994944 total_nvlink_bytes=2030239744 weight_to_token_ratio=0.080349 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382765427e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130044 payload_rows_remote=113660 dup_rows=1028 dup_rows_remote=978 dedup_ratio=0.007843 router_tokens_per_expert=[0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8185, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16216, 16306, 16374, 15616, 16384, 16380, 16384, 16384] dup_rows_per_dst=[14, 0, 118, 0, 2, 844, 0, 50] combine_rows_per_src=[16216, 16306, 16374, 15616, 16384, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=868073472 dispatch_meta_bytes=434712 combine_payload_bytes=868073472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=868508184 ingress_bytes=943570944 link_bytes=943570944 token_bytes=1736581656 weight_movement_bytes=75497472 total_nvlink_bytes=1812079128 weight_to_token_ratio=0.043475 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=62592 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.196580775e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458856 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=939982952 ingress_bytes=1317011456 link_bytes=1317011456 token_bytes=1879507048 weight_movement_bytes=377487360 total_nvlink_bytes=2256994408 weight_to_token_ratio=0.200844 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.670155910e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21937192704 prefetch_bytes=905969664 reduce_bytes=528482304 weight_movement_bytes=1434451968 egress_bytes=10971332352 ingress_bytes=12400312320 link_bytes=12400312320 total_nvlink_bytes=23371644672 weight_to_token_ratio=0.065389 prefetch_share_of_ingress=0.073060 dedup_ratio=0.024899 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=7 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049939 gemm_inner_pad_upper_bound=10668 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.310445125e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938213376 dispatch_meta_bytes=461920 combine_payload_bytes=938213376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938675296 ingress_bytes=1013710848 link_bytes=1013710848 token_bytes=1876888672 weight_movement_bytes=75497472 total_nvlink_bytes=1952386144 weight_to_token_ratio=0.040225 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285528046e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=939900928 ingress_bytes=1090437120 link_bytes=1090437120 token_bytes=1879343104 weight_movement_bytes=150994944 total_nvlink_bytes=2030338048 weight_to_token_ratio=0.080345 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382827759e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21925366688 prefetch_bytes=754974720 reduce_bytes=905969664 weight_movement_bytes=1660944384 egress_bytes=10965437344 ingress_bytes=12620873728 link_bytes=12620873728 total_nvlink_bytes=23586311072 weight_to_token_ratio=0.075754 prefetch_share_of_ingress=0.059820 dedup_ratio=0.029029 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=12 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049173 gemm_inner_pad_upper_bound=11557 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.333753701e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=931102720 dispatch_meta_bytes=458552 combine_payload_bytes=931102720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=931561272 ingress_bytes=1006600192 link_bytes=1006600192 token_bytes=1862663992 weight_movement_bytes=75497472 total_nvlink_bytes=1938161464 weight_to_token_ratio=0.040532 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.276510733e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21658513616 prefetch_bytes=981467136 reduce_bytes=377487360 weight_movement_bytes=1358954496 egress_bytes=10831999184 ingress_bytes=12185468928 link_bytes=12185468928 total_nvlink_bytes=23017468112 weight_to_token_ratio=0.062745 prefetch_share_of_ingress=0.080544 dedup_ratio=0.035651 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=5 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049479 gemm_inner_pad_upper_bound=11303 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.287740820e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21846119304 prefetch_bytes=981467136 reduce_bytes=452984832 weight_movement_bytes=1434451968 egress_bytes=10925806472 ingress_bytes=12354764800 link_bytes=12354764800 total_nvlink_bytes=23280571272 weight_to_token_ratio=0.065662 prefetch_share_of_ingress=0.079440 dedup_ratio=0.028722 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=6 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050245 gemm_inner_pad_upper_bound=10541 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.305631736e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21251261472 prefetch_bytes=981467136 reduce_bytes=603979776 weight_movement_bytes=1585446912 egress_bytes=10628334624 ingress_bytes=12208373760 link_bytes=12208373760 total_nvlink_bytes=22836708384 weight_to_token_ratio=0.074605 prefetch_share_of_ingress=0.080393 dedup_ratio=0.051398 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=8 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049939 gemm_inner_pad_upper_bound=10922 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.290161366e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114632 dup_rows=56 dup_rows_remote=20 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16368, 16384, 16364, 16384, 16364, 16384] dup_rows_per_dst=[36, 16, 4, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16368, 16384, 16364, 16384, 16364, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131010 payload_rows_remote=114626 dup_rows=62 dup_rows_remote=62 dedup_ratio=0.000473 router_tokens_per_expert=[0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8182, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8182, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16368, 16382, 16384, 16340, 16384, 16384] dup_rows_per_dst=[48, 0, 0, 2, 12, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16368, 16382, 16384, 16340, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458608 combine_payload_bytes=939065344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939523952 ingress_bytes=1014562816 link_bytes=1014562816 token_bytes=1878589296 weight_movement_bytes=75497472 total_nvlink_bytes=1954086768 weight_to_token_ratio=0.040188 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286608461e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939016192 dispatch_meta_bytes=458752 combine_payload_bytes=939016192 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939474944 ingress_bytes=1090011136 link_bytes=1090011136 token_bytes=1878491136 weight_movement_bytes=150994944 total_nvlink_bytes=2029486080 weight_to_token_ratio=0.080381 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382287551e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21194426960 prefetch_bytes=830472192 reduce_bytes=1434451968 weight_movement_bytes=2264924160 egress_bytes=10599909968 ingress_bytes=12859441152 link_bytes=12859441152 total_nvlink_bytes=23459351120 weight_to_token_ratio=0.106864 prefetch_share_of_ingress=0.064581 dedup_ratio=0.052783 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=19 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048866 gemm_inner_pad_upper_bound=12065 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.358965124e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21802176688 prefetch_bytes=528482304 reduce_bytes=1811939328 weight_movement_bytes=2340421632 egress_bytes=10903802032 ingress_bytes=13238796288 link_bytes=13238796288 total_nvlink_bytes=24142598320 weight_to_token_ratio=0.107348 prefetch_share_of_ingress=0.039919 dedup_ratio=0.029297 prefetch_slots=14 unused_prefetch_slots=370 slot_utilization=0.036458 reduce_slots=24 remote_demand=7 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049786 gemm_inner_pad_upper_bound=11176 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.399054766e-04 prefetch_bytes_per_gemm_flop=5.584916258e-06 anomalies=none + [2026-08-29 23:35:12] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 369.6 | throughput per GPU (TFLOP/s/GPU): 489.3 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.026910E+01 | loss scale: 1.0 | grad norm: 6.319 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=103986 payload_rows_remote=88802 dup_rows=27086 dup_rows_remote=23296 dedup_ratio=0.206650 router_tokens_per_expert=[0, 0, 0, 0, 0, 15, 0, 0, 0, 7430, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 605, 0, 2, 738, 4462, 0, 0, 8189, 4, 707, 1, 0, 0, 0, 0, 0, 1, 0, 0, 2, 1, 0, 0, 2057, 0, 0, 0, 7426, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2387, 0, 1, 0, 0, 8190, 0, 0, 0, 150, 0, 1, 0, 963, 0, 3909, 4600, 2, 0, 31, 0, 0, 1, 1, 3, 1, 0, 0, 2, 0, 2365, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4883, 6, 0, 3, 1, 1, 0, 34, 0, 0, 0, 0, 6343, 0, 1, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[14878, 12728, 15184, 8230, 13464, 14066, 10342, 15094] dup_rows_per_dst=[12, 3162, 3790, 1154, 7420, 3724, 1236, 6588] combine_rows_per_src=[14878, 12728, 15184, 8230, 13464, 14066, 10342, 15094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105614 payload_rows_remote=91898 dup_rows=25458 dup_rows_remote=20630 dedup_ratio=0.194229 router_tokens_per_expert=[1, 0, 0, 1, 0, 90, 1, 0, 0, 7794, 0, 2, 0, 2, 0, 8, 0, 6, 1, 0, 823, 0, 0, 1013, 3994, 0, 0, 8187, 11, 1295, 0, 0, 0, 0, 0, 0, 1, 8, 0, 0, 0, 0, 0, 1665, 0, 0, 23, 7899, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 1, 8183, 0, 0, 0, 63, 0, 0, 0, 922, 0, 3170, 5032, 0, 0, 86, 0, 0, 2, 0, 0, 0, 0, 2, 4, 0, 1807, 0, 0, 1, 2, 29, 0, 0, 2, 0, 0, 1, 0, 0, 0, 4, 0, 5242, 0, 0, 2, 1, 0, 0, 18, 0, 0, 1, 1, 5254, 0, 0, 0, 0, 3, 0, 1, 0, 0] payload_rows_per_dst=[15662, 5604, 15926, 16376, 7698, 16380, 14252, 13716] dup_rows_per_dst=[136, 694, 3266, 5754, 784, 3680, 6316, 4828] combine_rows_per_src=[15662, 5604, 15926, 16376, 7698, 16380, 14252, 13716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104256 payload_rows_remote=87876 dup_rows=26816 dup_rows_remote=21916 dedup_ratio=0.204590 router_tokens_per_expert=[0, 0, 0, 2, 5, 2, 0, 0, 0, 7763, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 735, 0, 1, 1021, 4422, 0, 0, 8192, 7, 1338, 1, 0, 0, 0, 0, 0, 1, 11, 0, 0, 0, 0, 0, 1682, 0, 0, 0, 7880, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3450, 0, 0, 0, 1, 8188, 0, 0, 0, 47, 0, 0, 0, 857, 0, 2450, 5116, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2438, 0, 0, 0, 0, 14, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 4026, 5, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 5804, 0, 0, 0, 0, 8, 0, 0, 0, 0] payload_rows_per_dst=[15536, 5716, 15946, 16384, 6296, 16380, 13580, 14418] dup_rows_per_dst=[16, 490, 3202, 6906, 530, 4900, 4722, 6050] combine_rows_per_src=[15536, 5716, 15946, 16384, 6296, 16380, 13580, 14418] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105668 payload_rows_remote=91256 dup_rows=25404 dup_rows_remote=18024 dedup_ratio=0.193817 router_tokens_per_expert=[0, 0, 0, 1, 0, 17, 0, 0, 0, 7870, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 833, 0, 0, 1089, 3654, 0, 0, 8192, 15, 674, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 2, 1859, 0, 0, 8, 7988, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2892, 0, 0, 0, 0, 8192, 0, 0, 0, 61, 0, 0, 0, 835, 0, 4408, 5509, 0, 0, 23, 0, 0, 0, 0, 0, 7, 0, 0, 2, 0, 2229, 0, 0, 0, 0, 15, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 5382, 2, 0, 1, 0, 0, 0, 9, 0, 0, 0, 0, 3758, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15748, 4720, 16074, 16384, 9716, 16384, 14412, 12230] dup_rows_per_dst=[30, 508, 3646, 5784, 938, 4506, 7380, 2612] combine_rows_per_src=[15748, 4720, 16074, 16384, 9716, 16384, 14412, 12230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104690 payload_rows_remote=88306 dup_rows=26382 dup_rows_remote=20950 dedup_ratio=0.201279 router_tokens_per_expert=[0, 0, 0, 0, 0, 12, 0, 0, 0, 7416, 0, 0, 0, 0, 0, 8, 0, 1, 0, 0, 861, 0, 0, 1013, 4996, 0, 0, 8192, 1, 610, 0, 0, 0, 0, 2, 0, 0, 18, 0, 2, 0, 0, 0, 1673, 0, 0, 3, 7722, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2716, 0, 0, 0, 0, 8192, 0, 0, 0, 93, 0, 0, 0, 911, 0, 4175, 5251, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 2834, 0, 0, 1, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5040, 24, 0, 3, 0, 0, 0, 10, 0, 0, 0, 0, 3659, 0, 0, 0, 0, 5, 0, 0, 0, 0] payload_rows_per_dst=[14858, 4490, 15600, 16384, 9374, 16384, 14066, 13534] dup_rows_per_dst=[14, 482, 3240, 5432, 1036, 5802, 6570, 3806] combine_rows_per_src=[14858, 4490, 15600, 16384, 9374, 16384, 14066, 13534] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=106704 payload_rows_remote=96430 dup_rows=24368 dup_rows_remote=23312 dedup_ratio=0.185913 router_tokens_per_expert=[0, 0, 0, 0, 0, 8, 0, 0, 0, 7789, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 903, 0, 0, 956, 4309, 0, 0, 8192, 1, 410, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1302, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2809, 0, 0, 0, 0, 8191, 0, 0, 0, 69, 0, 0, 0, 1025, 0, 4507, 5247, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2392, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4511, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 4788, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[15580, 4096, 16100, 16384, 10274, 16384, 14026, 13860] dup_rows_per_dst=[16, 450, 2568, 5618, 1056, 4792, 5490, 4378] combine_rows_per_src=[15580, 4096, 16100, 16384, 10274, 16384, 14026, 13860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=752828416 dispatch_meta_bytes=450112 combine_payload_bytes=752828416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=753278528 ingress_bytes=828325888 link_bytes=828325888 token_bytes=1506106944 weight_movement_bytes=75497472 total_nvlink_bytes=1581604416 weight_to_token_ratio=0.050128 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=39168 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.050433822e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=727465984 dispatch_meta_bytes=448392 combine_payload_bytes=727465984 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=727914376 ingress_bytes=878460928 link_bytes=878460928 token_bytes=1455380360 weight_movement_bytes=150994944 total_nvlink_bytes=1606375304 weight_to_token_ratio=0.103749 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=49792 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.114012109e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=719880192 dispatch_meta_bytes=439168 combine_payload_bytes=719880192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=720319360 ingress_bytes=795377664 link_bytes=795377664 token_bytes=1440199552 weight_movement_bytes=75497472 total_nvlink_bytes=1515697024 weight_to_token_ratio=0.052422 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=47488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.008650836e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=747569152 dispatch_meta_bytes=437120 combine_payload_bytes=747569152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=748006272 ingress_bytes=823066624 link_bytes=823066624 token_bytes=1495575424 weight_movement_bytes=75497472 total_nvlink_bytes=1571072896 weight_to_token_ratio=0.050481 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=38528 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.043764335e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=80196 payload_rows_remote=64300 dup_rows=50876 dup_rows_remote=44672 dedup_ratio=0.388153 router_tokens_per_expert=[0, 1, 1, 1, 4, 17, 0, 0, 0, 7828, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 860, 0, 0, 882, 4244, 0, 0, 8190, 6, 1722, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1842, 0, 2, 0, 7917, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2667, 0, 0, 0, 0, 8188, 0, 0, 0, 17, 0, 0, 0, 935, 0, 3195, 5728, 1, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1764, 1, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4701, 6, 0, 1, 0, 0, 0, 3, 0, 2, 1, 0, 4776, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15896, 16382, 3684, 5336, 16384, 3544, 9412, 9558] dup_rows_per_dst=[6204, 31260, 10, 0, 13390, 0, 4, 8] combine_rows_per_src=[15896, 16382, 3684, 5336, 16384, 3544, 9412, 9558] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=723402752 dispatch_meta_bytes=437024 combine_payload_bytes=723402752 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=723839776 ingress_bytes=798900224 link_bytes=798900224 token_bytes=1447242528 weight_movement_bytes=75497472 total_nvlink_bytes=1522740000 weight_to_token_ratio=0.052166 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=43264 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.013117938e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=789954560 dispatch_meta_bytes=478968 combine_payload_bytes=789954560 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=790433528 ingress_bytes=1091944448 link_bytes=1091944448 token_bytes=1580388088 weight_movement_bytes=301989888 total_nvlink_bytes=1882377976 weight_to_token_ratio=0.191086 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=24064 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.384739263e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87848 payload_rows_remote=71464 dup_rows=43224 dup_rows_remote=19318 dedup_ratio=0.329773 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 7893, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 630, 0, 0, 1069, 4615, 0, 0, 8191, 27, 1347, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1617, 0, 0, 0, 7509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2473, 0, 0, 0, 0, 8186, 0, 0, 0, 64, 0, 0, 0, 1007, 0, 3845, 5626, 0, 0, 51, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2062, 0, 0, 0, 0, 10, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4622, 8, 0, 1, 0, 0, 0, 5, 1, 0, 0, 0, 4668, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[15862, 16384, 10964, 4946, 16382, 4144, 9256, 9910] dup_rows_per_dst=[624, 23906, 1872, 0, 16330, 2, 8, 482] combine_rows_per_src=[15862, 16384, 10964, 4946, 16382, 4144, 9256, 9910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114660 payload_rows_remote=98278 dup_rows=16412 dup_rows_remote=5450 dedup_ratio=0.125214 router_tokens_per_expert=[8190, 1, 16, 2, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8042, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8189, 0, 0, 0, 0, 1, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2698, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5626, 0, 0, 0, 0] payload_rows_per_dst=[16336, 44, 16376, 16378, 16380, 16384, 16380, 16382] dup_rows_per_dst=[30, 2, 10, 10, 5396, 0, 2, 10962] combine_rows_per_src=[16336, 44, 16376, 16378, 16380, 16384, 16380, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129446 payload_rows_remote=113062 dup_rows=1626 dup_rows_remote=1626 dedup_ratio=0.012405 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 8192, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7384, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16382, 16384, 16384, 16384, 16384, 16384, 14768] dup_rows_per_dst=[0, 0, 0, 0, 1616, 6, 4, 0] combine_rows_per_src=[16376, 16382, 16384, 16384, 16384, 16384, 16384, 14768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=526745600 dispatch_meta_bytes=435888 combine_payload_bytes=526745600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=527181488 ingress_bytes=602243072 link_bytes=602243072 token_bytes=1053927088 weight_movement_bytes=75497472 total_nvlink_bytes=1129424560 weight_to_token_ratio=0.071634 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.034926 gemm_active_groups=16 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=7.637289874e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117442 payload_rows_remote=101058 dup_rows=13630 dup_rows_remote=13630 dedup_ratio=0.103989 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5128, 0, 0, 0, 0] payload_rows_per_dst=[16384, 4758, 16384, 16384, 16384, 16384, 16384, 14380] dup_rows_per_dst=[0, 0, 0, 0, 6128, 0, 0, 7502] combine_rows_per_src=[16384, 4758, 16384, 16384, 16384, 16384, 16384, 14380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129844 payload_rows_remote=113460 dup_rows=1228 dup_rows_remote=1228 dedup_ratio=0.009369 router_tokens_per_expert=[8192, 0, 4, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 610, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7575, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16384, 16384, 16384, 15160] dup_rows_per_dst=[4, 4, 0, 0, 1220, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16382, 16384, 16384, 16384, 15160] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128120 payload_rows_remote=111736 dup_rows=2952 dup_rows_remote=2952 dedup_ratio=0.022522 router_tokens_per_expert=[8192, 0, 8, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6716, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 13432] dup_rows_per_dst=[18, 4, 0, 0, 2930, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 13432] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=805093376 dispatch_meta_bytes=414912 combine_payload_bytes=805093376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=805508288 ingress_bytes=880590848 link_bytes=880590848 token_bytes=1610601664 weight_movement_bytes=75497472 total_nvlink_bytes=1686099136 weight_to_token_ratio=0.046875 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=51712 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116713148e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=926203904 dispatch_meta_bytes=458752 combine_payload_bytes=926203904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=926662656 ingress_bytes=1001701376 link_bytes=1001701376 token_bytes=1852866560 weight_movement_bytes=75497472 total_nvlink_bytes=1928364032 weight_to_token_ratio=0.040746 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.270298345e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=585433088 dispatch_meta_bytes=363128 combine_payload_bytes=585433088 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=585796216 ingress_bytes=811925504 link_bytes=811925504 token_bytes=1171229304 weight_movement_bytes=226492416 total_nvlink_bytes=1397721720 weight_to_token_ratio=0.193380 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.044118 gemm_active_groups=13 gemm_peak_group_rows=22272 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.029635826e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128970 payload_rows_remote=112586 dup_rows=2102 dup_rows_remote=4 dedup_ratio=0.016037 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7142, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 14284] dup_rows_per_dst=[0, 2, 0, 0, 2098, 2, 0, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16384, 14284] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=827867136 dispatch_meta_bytes=458752 combine_payload_bytes=827867136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=828325888 ingress_bytes=903364608 link_bytes=903364608 token_bytes=1656193024 weight_movement_bytes=75497472 total_nvlink_bytes=1731690496 weight_to_token_ratio=0.045585 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.145593481e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=929464320 dispatch_meta_bytes=458752 combine_payload_bytes=929464320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=929923072 ingress_bytes=1004961792 link_bytes=1004961792 token_bytes=1859387392 weight_movement_bytes=75497472 total_nvlink_bytes=1934884864 weight_to_token_ratio=0.040603 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.274433011e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=915341312 dispatch_meta_bytes=458752 combine_payload_bytes=915341312 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=915800064 ingress_bytes=1066336256 link_bytes=1066336256 token_bytes=1831141376 weight_movement_bytes=150994944 total_nvlink_bytes=1982136320 weight_to_token_ratio=0.082459 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.352264471e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=922304512 dispatch_meta_bytes=450360 combine_payload_bytes=922304512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=922754872 ingress_bytes=997801984 link_bytes=997801984 token_bytes=1845059384 weight_movement_bytes=75497472 total_nvlink_bytes=1920556856 weight_to_token_ratio=0.040919 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=51584 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265353367e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114672 payload_rows_remote=98288 dup_rows=16400 dup_rows_remote=16394 dedup_ratio=0.125122 router_tokens_per_expert=[8191, 0, 0, 1, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8190, 8191, 1, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1705, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 6483, 0, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 16382, 16382, 3412, 16376, 16382, 12970] dup_rows_per_dst=[6, 16390, 0, 0, 0, 0, 0, 4] combine_rows_per_src=[16384, 16384, 16382, 16382, 3412, 16376, 16382, 12970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127092 payload_rows_remote=110716 dup_rows=3980 dup_rows_remote=3878 dedup_ratio=0.030365 router_tokens_per_expert=[0, 3, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 756, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8188, 1, 0, 0, 2, 6485, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 1177, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 4, 0, 0, 0, 0, 1, 0, 0, 7958, 0, 0, 8184, 0, 0, 0, 8188, 0, 3, 0, 0, 1, 0, 1, 0, 8189, 0, 0] payload_rows_per_dst=[16282, 16174, 14236, 15330, 16384, 16380, 15930, 16376] dup_rows_per_dst=[8, 0, 1512, 1680, 6, 0, 672, 102] combine_rows_per_src=[16282, 16174, 14236, 15330, 16384, 16380, 15930, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130966 payload_rows_remote=114618 dup_rows=106 dup_rows_remote=106 dedup_ratio=0.000809 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7977, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16348, 16316, 16384, 16384, 16382, 16384] dup_rows_per_dst=[52, 54, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16348, 16316, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=805175296 dispatch_meta_bytes=458728 combine_payload_bytes=805175296 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=805634024 ingress_bytes=1107165184 link_bytes=1107165184 token_bytes=1610809320 weight_movement_bytes=301989888 total_nvlink_bytes=1912799208 weight_to_token_ratio=0.187477 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.404041299e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117884 payload_rows_remote=101500 dup_rows=13188 dup_rows_remote=1620 dedup_ratio=0.100616 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2577, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5614, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 8352, 16384, 16384, 11228] dup_rows_per_dst=[0, 11568, 0, 0, 1620, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 8352, 16384, 16384, 11228] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131052 payload_rows_remote=114668 dup_rows=20 dup_rows_remote=20 dedup_ratio=0.000153 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16368, 16384, 16384, 16384, 16380] dup_rows_per_dst=[6, 14, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16368, 16384, 16384, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114688 dup_rows=34 dup_rows_remote=34 dedup_ratio=0.000259 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16350, 16384, 16384, 16384, 16384] dup_rows_per_dst=[14, 20, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16350, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131024 payload_rows_remote=114640 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7505, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 681, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8175, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16372, 16384, 16384, 16384, 16350] dup_rows_per_dst=[2, 46, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16372, 16384, 16384, 16384, 16350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=906985472 dispatch_meta_bytes=458376 combine_payload_bytes=906985472 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=907443848 ingress_bytes=1208975360 link_bytes=1208975360 token_bytes=1814429320 weight_movement_bytes=301989888 total_nvlink_bytes=2116419208 weight_to_token_ratio=0.166438 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.533150933e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=938950656 dispatch_meta_bytes=458896 combine_payload_bytes=938950656 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939409552 ingress_bytes=1240940544 link_bytes=1240940544 token_bytes=1878360208 weight_movement_bytes=301989888 total_nvlink_bytes=2180350096 weight_to_token_ratio=0.160773 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.573687285e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131054 payload_rows_remote=114670 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000137 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16366, 16384, 16384, 16384, 16384] dup_rows_per_dst=[10, 8, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16366, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=939360256 dispatch_meta_bytes=458752 combine_payload_bytes=939360256 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939819008 ingress_bytes=1014857728 link_bytes=1014857728 token_bytes=1879179264 weight_movement_bytes=75497472 total_nvlink_bytes=1954676736 weight_to_token_ratio=0.040176 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286982451e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458888 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939982984 ingress_bytes=1015021568 link_bytes=1015021568 token_bytes=1879507080 weight_movement_bytes=75497472 total_nvlink_bytes=1955004552 weight_to_token_ratio=0.040169 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=61440 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287190223e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=939130880 dispatch_meta_bytes=458752 combine_payload_bytes=939130880 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939589632 ingress_bytes=1090125824 link_bytes=1090125824 token_bytes=1878720512 weight_movement_bytes=150994944 total_nvlink_bytes=2029715456 weight_to_token_ratio=0.080371 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382432992e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=831488000 dispatch_meta_bytes=412480 combine_payload_bytes=831488000 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=831900480 ingress_bytes=1057980416 link_bytes=1057980416 token_bytes=1663388480 weight_movement_bytes=226492416 total_nvlink_bytes=1889880896 weight_to_token_ratio=0.136163 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=51584 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.341668089e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=939376640 dispatch_meta_bytes=458752 combine_payload_bytes=939376640 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939835392 ingress_bytes=1014874112 link_bytes=1014874112 token_bytes=1879212032 weight_movement_bytes=75497472 total_nvlink_bytes=1954709504 weight_to_token_ratio=0.040175 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287003228e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130332 payload_rows_remote=113948 dup_rows=740 dup_rows_remote=734 dedup_ratio=0.005646 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 336, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 7103, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 859, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8084, 0, 0, 8189, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 15924, 16376, 16384, 16168, 16330] dup_rows_per_dst=[6, 48, 680, 0, 2, 4, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 15924, 16376, 16384, 16168, 16330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114660 dup_rows=28 dup_rows_remote=26 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 16384, 16360, 16384] dup_rows_per_dst=[0, 26, 2, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 16384, 16360, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130868 payload_rows_remote=114490 dup_rows=204 dup_rows_remote=188 dedup_ratio=0.001556 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8180, 0, 3, 2, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 1, 8188, 8187, 0, 0, 0, 0, 0, 0, 8188, 8185, 0, 0, 3] payload_rows_per_dst=[16378, 16264, 16366, 16380, 16366, 16380, 16356, 16378] dup_rows_per_dst=[2, 46, 6, 6, 42, 4, 82, 16] combine_rows_per_src=[16378, 16264, 16366, 16380, 16366, 16380, 16356, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114688 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[0, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=933462016 dispatch_meta_bytes=458728 combine_payload_bytes=933462016 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933920744 ingress_bytes=1008959488 link_bytes=1008959488 token_bytes=1867382760 weight_movement_bytes=75497472 total_nvlink_bytes=1942880232 weight_to_token_ratio=0.040430 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.279502653e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131042 payload_rows_remote=114658 dup_rows=30 dup_rows_remote=30 dedup_ratio=0.000229 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16378, 16384, 16360, 16384] dup_rows_per_dst=[0, 22, 8, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16378, 16384, 16360, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939294720 dispatch_meta_bytes=458744 combine_payload_bytes=939294720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939753464 ingress_bytes=1014792192 link_bytes=1014792192 token_bytes=1879048184 weight_movement_bytes=75497472 total_nvlink_bytes=1954545656 weight_to_token_ratio=0.040179 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286899342e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=937902080 dispatch_meta_bytes=458712 combine_payload_bytes=937902080 prefetch_bytes=0 reduce_bytes=452984832 egress_bytes=938360792 ingress_bytes=1390886912 link_bytes=1390886912 token_bytes=1876262872 weight_movement_bytes=452984832 total_nvlink_bytes=2329247704 weight_to_token_ratio=0.241429 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.763840387e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131048 payload_rows_remote=114664 dup_rows=24 dup_rows_remote=4 dedup_ratio=0.000183 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16380, 16366, 16384, 16384, 16384, 16382] dup_rows_per_dst=[4, 20, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 16366, 16384, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458768 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939982864 ingress_bytes=1015021568 link_bytes=1015021568 token_bytes=1879506960 weight_movement_bytes=75497472 total_nvlink_bytes=1955004432 weight_to_token_ratio=0.040169 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287190223e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939278336 dispatch_meta_bytes=458752 combine_payload_bytes=939278336 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939737088 ingress_bytes=1090273280 link_bytes=1090273280 token_bytes=1879015424 weight_movement_bytes=150994944 total_nvlink_bytes=2030010368 weight_to_token_ratio=0.080359 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382619987e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=0 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=939524096 link_bytes=939982848 token_bytes=1879506944 weight_movement_bytes=0 total_nvlink_bytes=1879506944 weight_to_token_ratio=0.000000 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.192030564e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939966464 ingress_bytes=1166000128 link_bytes=1166000128 token_bytes=1879474176 weight_movement_bytes=226492416 total_nvlink_bytes=2105966592 weight_to_token_ratio=0.120508 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478652289e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=939327488 dispatch_meta_bytes=458672 combine_payload_bytes=939327488 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939786160 ingress_bytes=1090322432 link_bytes=1090322432 token_bytes=1879113648 weight_movement_bytes=150994944 total_nvlink_bytes=2030108592 weight_to_token_ratio=0.080354 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382682318e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131038 payload_rows_remote=114656 dup_rows=34 dup_rows_remote=28 dedup_ratio=0.000259 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 1, 1, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 1, 0, 8191, 8191, 0, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16382, 16368, 16382, 16384, 16372] dup_rows_per_dst=[6, 18, 2, 0, 4, 2, 0, 2] combine_rows_per_src=[16382, 16384, 16384, 16382, 16368, 16382, 16384, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127434 payload_rows_remote=111284 dup_rows=3638 dup_rows_remote=3604 dedup_ratio=0.027756 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 8170, 6567, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 8075, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1798, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[13140, 16384, 16150, 16384, 16384, 16382, 16232, 16378] dup_rows_per_dst=[0, 0, 34, 2, 3596, 0, 0, 6] combine_rows_per_src=[13140, 16384, 16150, 16384, 16384, 16382, 16232, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130186 payload_rows_remote=113804 dup_rows=886 dup_rows_remote=878 dedup_ratio=0.006760 router_tokens_per_expert=[0, 0, 8189, 0, 0, 8180, 0, 0, 3, 0, 0, 0, 0, 8187, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 21, 0, 0, 0, 41, 1, 3, 0, 0, 4, 0, 8154, 0, 0, 0, 2, 0, 0, 0, 4, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8188, 8149, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8188, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16356, 16340, 16380, 16368, 15594, 16384, 16382, 16382] dup_rows_per_dst=[0, 42, 30, 34, 2, 8, 762, 8] combine_rows_per_src=[16356, 16340, 16380, 16368, 15594, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130176 payload_rows_remote=113862 dup_rows=896 dup_rows_remote=896 dedup_ratio=0.006836 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8189, 7809, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 445, 1, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15618, 16384, 16328, 16384, 16384, 16384, 16314, 16380] dup_rows_per_dst=[0, 0, 0, 0, 890, 0, 0, 6] combine_rows_per_src=[15618, 16384, 16328, 16384, 16384, 16384, 16314, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129860 payload_rows_remote=113478 dup_rows=1212 dup_rows_remote=1212 dedup_ratio=0.009247 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 8189, 7867, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7962, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 600, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15736, 16384, 15926, 16384, 16384, 16382, 16280, 16384] dup_rows_per_dst=[0, 0, 2, 0, 1200, 0, 0, 10] combine_rows_per_src=[15736, 16384, 15926, 16384, 16384, 16382, 16280, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939261952 dispatch_meta_bytes=458736 combine_payload_bytes=939261952 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939720688 ingress_bytes=1090256896 link_bytes=1090256896 token_bytes=1878982640 weight_movement_bytes=150994944 total_nvlink_bytes=2029977584 weight_to_token_ratio=0.080360 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382599209e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128738 payload_rows_remote=112354 dup_rows=2334 dup_rows_remote=8 dedup_ratio=0.017807 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8187, 7327, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1164, 0, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8124, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14654, 16384, 15920, 16384, 16384, 16384, 16248, 16380] dup_rows_per_dst=[0, 0, 8, 0, 2326, 0, 0, 0] combine_rows_per_src=[14654, 16384, 15920, 16384, 16384, 16384, 16248, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130112 payload_rows_remote=113728 dup_rows=960 dup_rows_remote=960 dedup_ratio=0.007324 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8190, 7906, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 480, 0, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15812, 16384, 16138, 16384, 16384, 16384, 16246, 16380] dup_rows_per_dst=[0, 0, 4, 0, 956, 0, 0, 0] combine_rows_per_src=[15812, 16384, 16138, 16384, 16384, 16384, 16246, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=911638528 dispatch_meta_bytes=459552 combine_payload_bytes=911638528 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=912098080 ingress_bytes=987136000 link_bytes=987136000 token_bytes=1823736608 weight_movement_bytes=75497472 total_nvlink_bytes=1899234080 weight_to_token_ratio=0.041397 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.251827398e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=932757504 dispatch_meta_bytes=459032 combine_payload_bytes=932757504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933216536 ingress_bytes=1008254976 link_bytes=1008254976 token_bytes=1865974040 weight_movement_bytes=75497472 total_nvlink_bytes=1941471512 weight_to_token_ratio=0.040460 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278609232e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=932282368 dispatch_meta_bytes=458728 combine_payload_bytes=932282368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=932741096 ingress_bytes=1007779840 link_bytes=1007779840 token_bytes=1865023464 weight_movement_bytes=75497472 total_nvlink_bytes=1940520936 weight_to_token_ratio=0.040481 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278006693e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=929611776 dispatch_meta_bytes=458760 combine_payload_bytes=929611776 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=930070536 ingress_bytes=1080606720 link_bytes=1080606720 token_bytes=1859682312 weight_movement_bytes=150994944 total_nvlink_bytes=2010677256 weight_to_token_ratio=0.081194 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.370361428e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=920403968 dispatch_meta_bytes=449448 combine_payload_bytes=920403968 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=920853416 ingress_bytes=1222393856 link_bytes=1222393856 token_bytes=1841257384 weight_movement_bytes=301989888 total_nvlink_bytes=2143247272 weight_to_token_ratio=0.164013 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=60288 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.550167474e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=931659776 dispatch_meta_bytes=458752 combine_payload_bytes=931659776 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=932118528 ingress_bytes=1082654720 link_bytes=1082654720 token_bytes=1863778304 weight_movement_bytes=150994944 total_nvlink_bytes=2014773248 weight_to_token_ratio=0.081016 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.372958580e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458744 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939966456 ingress_bytes=1015005184 link_bytes=1015005184 token_bytes=1879474168 weight_movement_bytes=75497472 total_nvlink_bytes=1954971640 weight_to_token_ratio=0.040169 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287169446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121268 payload_rows_remote=104884 dup_rows=9804 dup_rows_remote=2048 dedup_ratio=0.074799 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8181, 0, 0, 1, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 1, 1, 0, 0, 31, 0, 8052, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 105, 0, 0, 8191, 8157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16252, 16364, 6752, 16382, 16384, 16366] dup_rows_per_dst=[7756, 2, 2034, 0, 10, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16252, 16364, 6752, 16382, 16384, 16366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117190 payload_rows_remote=100832 dup_rows=13882 dup_rows_remote=11044 dedup_ratio=0.105911 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1264, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 5516, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1423, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 2528, 16384, 16384, 16384, 16358, 16384] dup_rows_per_dst=[0, 0, 0, 12, 11032, 0, 2838, 0] combine_rows_per_src=[16384, 16384, 2528, 16384, 16384, 16384, 16358, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129470 payload_rows_remote=113086 dup_rows=1602 dup_rows_remote=520 dedup_ratio=0.012222 router_tokens_per_expert=[0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 541, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 7447, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8087, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 14894, 16384, 16284, 16372] dup_rows_per_dst=[8, 0, 1082, 18, 0, 28, 466, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 14894, 16384, 16284, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122600 payload_rows_remote=106216 dup_rows=8472 dup_rows_remote=8452 dedup_ratio=0.064636 router_tokens_per_expert=[0, 0, 8183, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4037, 0, 0, 0, 0, 8185, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 16, 0, 0, 3, 0, 8188, 0, 0, 0, 3, 8184, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 8188, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4138, 0, 0, 0, 8188, 2, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16370, 16378, 8080, 16298, 16382, 16332, 16376, 16384] dup_rows_per_dst=[0, 4, 0, 2, 118, 2, 8326, 20] combine_rows_per_src=[16370, 16378, 8080, 16298, 16382, 16332, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116786 payload_rows_remote=100402 dup_rows=14286 dup_rows_remote=14272 dedup_ratio=0.108994 router_tokens_per_expert=[0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 780, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 7151, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 379, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 2268, 16384, 16324, 16384, 16280, 16378] dup_rows_per_dst=[4, 0, 90, 32, 13564, 14, 582, 0] combine_rows_per_src=[16384, 16384, 2268, 16384, 16324, 16384, 16280, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128570 payload_rows_remote=114666 dup_rows=2502 dup_rows_remote=2502 dedup_ratio=0.019089 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1044, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 6952, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 201, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 13904, 16384, 16364, 16382] dup_rows_per_dst=[2, 0, 2088, 6, 0, 8, 398, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 13904, 16384, 16364, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=859209728 dispatch_meta_bytes=427728 combine_payload_bytes=859209728 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=859637456 ingress_bytes=1312194560 link_bytes=1312194560 token_bytes=1718847184 weight_movement_bytes=452984832 total_nvlink_bytes=2171832016 weight_to_token_ratio=0.263540 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=61696 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.664047407e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128242 payload_rows_remote=111858 dup_rows=2830 dup_rows_remote=2822 dedup_ratio=0.021591 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1064, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 6801, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 455, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 13602, 16384, 16336, 16384] dup_rows_per_dst=[0, 0, 2128, 8, 0, 2, 692, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 13602, 16384, 16336, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=826015744 dispatch_meta_bytes=447504 combine_payload_bytes=826015744 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=826463248 ingress_bytes=1052508160 link_bytes=1052508160 token_bytes=1652478992 weight_movement_bytes=226492416 total_nvlink_bytes=1878971408 weight_to_token_ratio=0.137062 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=56192 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.334728499e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=926400512 dispatch_meta_bytes=454424 combine_payload_bytes=926400512 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=926854936 ingress_bytes=1077395456 link_bytes=1077395456 token_bytes=1853255448 weight_movement_bytes=150994944 total_nvlink_bytes=2004250392 weight_to_token_ratio=0.081476 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=41472 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.366289093e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=870121472 dispatch_meta_bytes=458672 combine_payload_bytes=870121472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=870580144 ingress_bytes=945618944 link_bytes=945618944 token_bytes=1740701616 weight_movement_bytes=75497472 total_nvlink_bytes=1816199088 weight_to_token_ratio=0.043372 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.199177927e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129616 payload_rows_remote=113232 dup_rows=1456 dup_rows_remote=1456 dedup_ratio=0.011108 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8182, 7808, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 7931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 721, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15616, 16384, 15862, 16384, 16384, 16382, 16222, 16382] dup_rows_per_dst=[0, 0, 4, 0, 1442, 0, 0, 10] combine_rows_per_src=[15616, 16384, 15862, 16384, 16384, 16382, 16222, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=822493184 dispatch_meta_bytes=458696 combine_payload_bytes=822493184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=822951880 ingress_bytes=897990656 link_bytes=897990656 token_bytes=1645445064 weight_movement_bytes=75497472 total_nvlink_bytes=1720942536 weight_to_token_ratio=0.045883 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.138778554e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=939343872 dispatch_meta_bytes=468672 combine_payload_bytes=939343872 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939812544 ingress_bytes=1165836288 link_bytes=1165836288 token_bytes=1879156416 weight_movement_bytes=226492416 total_nvlink_bytes=2105648832 weight_to_token_ratio=0.120529 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=41344 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478444517e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=916340736 dispatch_meta_bytes=458720 combine_payload_bytes=916340736 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=916799456 ingress_bytes=991838208 link_bytes=991838208 token_bytes=1833140192 weight_movement_bytes=75497472 total_nvlink_bytes=1908637664 weight_to_token_ratio=0.041185 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.257790460e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=927596544 dispatch_meta_bytes=458752 combine_payload_bytes=927596544 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928055296 ingress_bytes=1003094016 link_bytes=1003094016 token_bytes=1855651840 weight_movement_bytes=75497472 total_nvlink_bytes=1931149312 weight_to_token_ratio=0.040685 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.272064408e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=103318 payload_rows_remote=86938 dup_rows=27754 dup_rows_remote=27752 dedup_ratio=0.211746 router_tokens_per_expert=[0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 5684, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 151, 0, 0, 1, 0, 8191, 0, 0, 0, 1, 8191, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2356, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16382, 310, 16382, 4712, 16384] dup_rows_per_dst=[2, 2, 27742, 4, 2, 0, 0, 2] combine_rows_per_src=[16380, 16384, 16384, 16382, 310, 16382, 4712, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119128 payload_rows_remote=102744 dup_rows=11944 dup_rows_remote=11944 dedup_ratio=0.091125 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3633, 0, 8192, 0, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6558, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5957] payload_rows_per_dst=[16384, 7720, 13116, 16384, 16384, 16372, 16384, 16384] dup_rows_per_dst=[0, 30, 0, 0, 0, 0, 0, 11914] combine_rows_per_src=[16384, 7720, 13116, 16384, 16384, 16372, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118870 payload_rows_remote=102646 dup_rows=12202 dup_rows_remote=4186 dedup_ratio=0.093094 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 8192, 0, 0, 5183, 0, 8192, 0, 0, 0, 0, 0, 407, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7490, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3333] payload_rows_per_dst=[16384, 14124, 16224, 16384, 16384, 16322, 16382, 6666] dup_rows_per_dst=[6, 4180, 8016, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 14124, 16224, 16384, 16384, 16322, 16382, 6666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125296 payload_rows_remote=108924 dup_rows=5776 dup_rows_remote=502 dedup_ratio=0.044067 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 8188, 3, 0, 7526, 0, 8188, 0, 2, 0, 4, 0, 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5982, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 4, 1, 0, 89, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 4, 0, 0, 0, 0, 3, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8003, 0, 0, 0, 0, 2819] payload_rows_per_dst=[16356, 15086, 11972, 16384, 16360, 16384, 16382, 16372] dup_rows_per_dst=[18, 288, 0, 196, 0, 0, 0, 5274] combine_rows_per_src=[16356, 15086, 11972, 16384, 16360, 16384, 16382, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123606 payload_rows_remote=107264 dup_rows=7466 dup_rows_remote=7466 dedup_ratio=0.056961 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4886, 0, 8192, 0, 0, 0, 0, 0, 735, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3639] payload_rows_per_dst=[16384, 11054, 14290, 16384, 16384, 16342, 16384, 16384] dup_rows_per_dst=[0, 188, 0, 0, 0, 0, 0, 7278] combine_rows_per_src=[16384, 11054, 14290, 16384, 16384, 16342, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124696 payload_rows_remote=108312 dup_rows=6376 dup_rows_remote=6376 dedup_ratio=0.048645 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5247, 0, 8192, 0, 0, 0, 0, 0, 304, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7659, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 3177] payload_rows_per_dst=[16384, 11080, 15318, 16384, 16384, 16378, 16384, 16384] dup_rows_per_dst=[0, 22, 0, 0, 0, 0, 0, 6354] combine_rows_per_src=[16384, 11080, 15318, 16384, 16384, 16378, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=712196096 dispatch_meta_bytes=458760 combine_payload_bytes=712196096 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=712654856 ingress_bytes=863191040 link_bytes=863191040 token_bytes=1424850952 weight_movement_bytes=150994944 total_nvlink_bytes=1575845896 weight_to_token_ratio=0.105972 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.094647742e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=841678848 dispatch_meta_bytes=458752 combine_payload_bytes=841678848 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=842137600 ingress_bytes=917176320 link_bytes=917176320 token_bytes=1683816448 weight_movement_bytes=75497472 total_nvlink_bytes=1759313920 weight_to_token_ratio=0.044837 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.163108676e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121184 payload_rows_remote=104800 dup_rows=9888 dup_rows_remote=9888 dedup_ratio=0.075439 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 5590, 0, 8192, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4924, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5692] payload_rows_per_dst=[16384, 11462, 16356, 16384, 16384, 16384, 16384, 11446] dup_rows_per_dst=[0, 74, 9774, 0, 0, 0, 0, 40] combine_rows_per_src=[16384, 11462, 16356, 16384, 16384, 16384, 16384, 11446] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=840876032 dispatch_meta_bytes=427328 combine_payload_bytes=840876032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=841303360 ingress_bytes=916373504 link_bytes=916373504 token_bytes=1682179392 weight_movement_bytes=75497472 total_nvlink_bytes=1757676864 weight_to_token_ratio=0.044881 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=52864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.162090592e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127678 payload_rows_remote=111294 dup_rows=3394 dup_rows_remote=3394 dedup_ratio=0.025894 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 748, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 7278, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 172, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 14556, 16384, 14820, 16382] dup_rows_per_dst=[2, 0, 3146, 6, 0, 0, 240, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 14556, 16384, 14820, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=892305408 dispatch_meta_bytes=437704 combine_payload_bytes=892305408 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=892743112 ingress_bytes=1269792768 link_bytes=1269792768 token_bytes=1785048520 weight_movement_bytes=377487360 total_nvlink_bytes=2162535880 weight_to_token_ratio=0.211472 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=33024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.610275967e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=878706688 dispatch_meta_bytes=458920 combine_payload_bytes=878706688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=879165608 ingress_bytes=954204160 link_bytes=954204160 token_bytes=1757872296 weight_movement_bytes=75497472 total_nvlink_bytes=1833369768 weight_to_token_ratio=0.042948 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.210065189e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=887291904 dispatch_meta_bytes=458752 combine_payload_bytes=887291904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=887750656 ingress_bytes=962789376 link_bytes=962789376 token_bytes=1775042560 weight_movement_bytes=75497472 total_nvlink_bytes=1850540032 weight_to_token_ratio=0.042533 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.220952452e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=858521600 dispatch_meta_bytes=458752 combine_payload_bytes=858521600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=858980352 ingress_bytes=934019072 link_bytes=934019072 token_bytes=1717501952 weight_movement_bytes=75497472 total_nvlink_bytes=1792999424 weight_to_token_ratio=0.043958 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.184467656e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=911720448 dispatch_meta_bytes=458752 combine_payload_bytes=911720448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=912179200 ingress_bytes=987217920 link_bytes=987217920 token_bytes=1823899648 weight_movement_bytes=75497472 total_nvlink_bytes=1899397120 weight_to_token_ratio=0.041393 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.251931284e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130616 payload_rows_remote=114232 dup_rows=456 dup_rows_remote=456 dedup_ratio=0.003479 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8184, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[15936, 16376, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 448, 0, 8, 0, 0, 0, 0] combine_rows_per_src=[15936, 16376, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118484 payload_rows_remote=102102 dup_rows=12588 dup_rows_remote=12582 dedup_ratio=0.096039 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 8191, 1, 0, 5695, 0, 8191, 0, 0, 0, 1, 0, 417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5915, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8121, 0, 0, 0, 0, 4375] payload_rows_per_dst=[16382, 16384, 11830, 16184, 16382, 16384, 16384, 8554] dup_rows_per_dst=[6, 12386, 0, 4, 26, 162, 4, 0] combine_rows_per_src=[16382, 16384, 11830, 16184, 16382, 16384, 16384, 8554] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130996 payload_rows_remote=114628 dup_rows=76 dup_rows_remote=76 dedup_ratio=0.000580 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8183, 8179, 0, 0, 8157, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16330, 16368, 16380, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 22, 0, 54, 0, 0, 0, 0] combine_rows_per_src=[16384, 16330, 16368, 16380, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114638 payload_rows_remote=98260 dup_rows=16434 dup_rows_remote=16410 dedup_ratio=0.125381 router_tokens_per_expert=[0, 8188, 0, 0, 0, 0, 0, 0, 7203, 0, 0, 0, 0, 0, 8188, 8188, 0, 0, 3640, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 4, 1, 0, 0, 5523, 0, 0, 0, 4, 0, 0, 0, 3, 0, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 2, 8184, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 4, 0, 3, 1, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 9, 0, 8188, 0] payload_rows_per_dst=[0, 16356, 16384, 16384, 16376, 16384, 16376, 16378] dup_rows_per_dst=[0, 5312, 24, 11060, 4, 4, 6, 24] combine_rows_per_src=[0, 16356, 16384, 16384, 16376, 16384, 16376, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114678 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 8187, 0, 0, 8181, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16380, 16384, 16378, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 6, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16384, 16378, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=935788544 dispatch_meta_bytes=458752 combine_payload_bytes=935788544 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=936247296 ingress_bytes=1086783488 link_bytes=1086783488 token_bytes=1872035840 weight_movement_bytes=150994944 total_nvlink_bytes=2023030784 weight_to_token_ratio=0.080658 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378194439e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 8176, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16380, 16380, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 4, 0, 4, 0, 0, 0, 0] combine_rows_per_src=[16384, 16380, 16380, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=836419584 dispatch_meta_bytes=458736 combine_payload_bytes=836419584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=836878320 ingress_bytes=911917056 link_bytes=911917056 token_bytes=1673297904 weight_movement_bytes=75497472 total_nvlink_bytes=1748795376 weight_to_token_ratio=0.045119 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.156439189e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 8174, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939032576 dispatch_meta_bytes=458816 combine_payload_bytes=939032576 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939491392 ingress_bytes=1090027520 link_bytes=1090027520 token_bytes=1878523968 weight_movement_bytes=150994944 total_nvlink_bytes=2029518912 weight_to_token_ratio=0.080380 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382308328e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120462 payload_rows_remote=104078 dup_rows=10610 dup_rows_remote=0 dedup_ratio=0.080948 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 4861, 0, 8192, 0, 0, 0, 0, 0, 444, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7088, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 4011] payload_rows_per_dst=[16384, 16384, 14176, 16384, 16384, 16344, 16384, 8022] dup_rows_per_dst=[0, 10610, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 14176, 16384, 16384, 16344, 16384, 8022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=804945920 dispatch_meta_bytes=458680 combine_payload_bytes=804945920 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=805404600 ingress_bytes=880443392 link_bytes=880443392 token_bytes=1610350520 weight_movement_bytes=75497472 total_nvlink_bytes=1685847992 weight_to_token_ratio=0.046883 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116526153e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939442176 dispatch_meta_bytes=458752 combine_payload_bytes=939442176 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=939900928 ingress_bytes=1014939648 link_bytes=1014939648 token_bytes=1879343104 weight_movement_bytes=75497472 total_nvlink_bytes=1954840576 weight_to_token_ratio=0.040172 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287086337e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939917312 ingress_bytes=1014956032 link_bytes=1014956032 token_bytes=1879375872 weight_movement_bytes=75497472 total_nvlink_bytes=1954873344 weight_to_token_ratio=0.040172 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287107114e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939966464 ingress_bytes=1015005184 link_bytes=1015005184 token_bytes=1879474176 weight_movement_bytes=75497472 total_nvlink_bytes=1954971648 weight_to_token_ratio=0.040169 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=56704 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287169446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=852606976 dispatch_meta_bytes=416312 combine_payload_bytes=852606976 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=853023288 ingress_bytes=1079099392 link_bytes=1079099392 token_bytes=1705630264 weight_movement_bytes=226492416 total_nvlink_bytes=1932122680 weight_to_token_ratio=0.132791 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=42624 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.368449924e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120826 payload_rows_remote=104472 dup_rows=10246 dup_rows_remote=10246 dedup_ratio=0.078171 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 8170, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 6172, 16384, 16380, 16384, 16384, 16354, 16384] dup_rows_per_dst=[2, 4, 0, 10168, 42, 0, 0, 30] combine_rows_per_src=[16384, 6172, 16384, 16380, 16384, 16384, 16354, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115696 payload_rows_remote=99314 dup_rows=15376 dup_rows_remote=14 dedup_ratio=0.117310 router_tokens_per_expert=[0, 8191, 0, 0, 0, 1, 0, 0, 7671, 0, 0, 0, 0, 0, 8189, 8191, 0, 0, 5237, 105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 3367, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0] payload_rows_per_dst=[16382, 10684, 16380, 6736, 16384, 16380, 16384, 16366] dup_rows_per_dst=[15362, 0, 2, 4, 4, 0, 4, 0] combine_rows_per_src=[16382, 10684, 16380, 6736, 16384, 16380, 16384, 16366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130896 payload_rows_remote=114512 dup_rows=176 dup_rows_remote=176 dedup_ratio=0.001343 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8163, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16328, 16384, 16384, 16384, 16384, 16264, 16384] dup_rows_per_dst=[62, 2, 0, 0, 78, 0, 0, 34] combine_rows_per_src=[16384, 16328, 16384, 16384, 16384, 16384, 16264, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 2, 1, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130840 payload_rows_remote=114456 dup_rows=232 dup_rows_remote=68 dedup_ratio=0.001770 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8155, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16314, 16384, 16384, 16384, 16384, 16222, 16384] dup_rows_per_dst=[6, 12, 0, 0, 164, 0, 0, 50] combine_rows_per_src=[16384, 16314, 16384, 16384, 16384, 16384, 16222, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130974 payload_rows_remote=114590 dup_rows=98 dup_rows_remote=98 dedup_ratio=0.000748 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8161, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16322, 16384, 16384, 16384, 16384, 16348, 16384] dup_rows_per_dst=[46, 0, 0, 0, 38, 0, 0, 14] combine_rows_per_src=[16384, 16322, 16384, 16384, 16384, 16384, 16348, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123478 payload_rows_remote=107100 dup_rows=7594 dup_rows_remote=7386 dedup_ratio=0.057938 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 0, 9, 0, 0, 0, 0, 0, 2, 0, 0, 2, 0, 0, 7938, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 626, 0, 0, 0, 0, 0, 104, 8175, 0, 0, 0, 0, 0, 591, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 8, 2, 0, 0, 0, 0, 8081, 1, 3, 0, 113, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 3, 4, 7928, 3, 276, 0, 0, 0, 0, 0, 0, 0, 2, 0] payload_rows_per_dst=[16214, 15886, 16374, 16376, 11632, 16182, 14436, 16378] dup_rows_per_dst=[18, 6, 12, 1434, 982, 1090, 3844, 208] combine_rows_per_src=[16214, 15886, 16374, 16376, 11632, 16182, 14436, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130984 payload_rows_remote=114600 dup_rows=88 dup_rows_remote=88 dedup_ratio=0.000671 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8166, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16362, 16384, 16384, 16380, 16384, 16322, 16384] dup_rows_per_dst=[22, 8, 0, 0, 36, 0, 0, 22] combine_rows_per_src=[16384, 16362, 16384, 16384, 16380, 16384, 16322, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=855834624 dispatch_meta_bytes=458872 combine_payload_bytes=855834624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=856293496 ingress_bytes=931332096 link_bytes=931332096 token_bytes=1712128120 weight_movement_bytes=75497472 total_nvlink_bytes=1787625592 weight_to_token_ratio=0.044096 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.181060193e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=813580288 dispatch_meta_bytes=397312 combine_payload_bytes=813580288 prefetch_bytes=75497472 reduce_bytes=452984832 egress_bytes=813977600 ingress_bytes=1342062592 link_bytes=1342062592 token_bytes=1627557888 weight_movement_bytes=528482304 total_nvlink_bytes=2156040192 weight_to_token_ratio=0.324709 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=56704 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.701924276e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938082304 dispatch_meta_bytes=458752 combine_payload_bytes=938082304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938541056 ingress_bytes=1013579776 link_bytes=1013579776 token_bytes=1876623360 weight_movement_bytes=75497472 total_nvlink_bytes=1952120832 weight_to_token_ratio=0.040230 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285361828e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129594 payload_rows_remote=113210 dup_rows=1478 dup_rows_remote=1474 dedup_ratio=0.011276 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8191, 8190, 0, 0, 8185, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16382, 14908, 16384, 16384, 16384, 16384] dup_rows_per_dst=[1474, 4, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 14908, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=937623552 dispatch_meta_bytes=458096 combine_payload_bytes=937623552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938081648 ingress_bytes=1013121024 link_bytes=1013121024 token_bytes=1875705200 weight_movement_bytes=75497472 total_nvlink_bytes=1951202672 weight_to_token_ratio=0.040250 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63104 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284780066e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938721280 dispatch_meta_bytes=458752 combine_payload_bytes=938721280 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939180032 ingress_bytes=1240711168 link_bytes=1240711168 token_bytes=1877901312 weight_movement_bytes=301989888 total_nvlink_bytes=2179891200 weight_to_token_ratio=0.160812 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.573396404e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=877363200 dispatch_meta_bytes=457944 combine_payload_bytes=877363200 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=877821144 ingress_bytes=1028358144 link_bytes=1028358144 token_bytes=1755184344 weight_movement_bytes=150994944 total_nvlink_bytes=1906179288 weight_to_token_ratio=0.086028 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.304102879e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938803200 dispatch_meta_bytes=458752 combine_payload_bytes=938803200 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939261952 ingress_bytes=1089798144 link_bytes=1089798144 token_bytes=1878065152 weight_movement_bytes=150994944 total_nvlink_bytes=2029060096 weight_to_token_ratio=0.080399 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=50816 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382017447e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=927416320 dispatch_meta_bytes=458736 combine_payload_bytes=927416320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=927875056 ingress_bytes=1002913792 link_bytes=1002913792 token_bytes=1855291376 weight_movement_bytes=75497472 total_nvlink_bytes=1930788848 weight_to_token_ratio=0.040693 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.056985 gemm_active_groups=4 gemm_peak_group_rows=57984 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.271835859e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116684 payload_rows_remote=100302 dup_rows=14388 dup_rows_remote=14384 dedup_ratio=0.109772 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 5496, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 75, 0, 0, 0, 0, 0, 1495, 8162, 0, 0, 0, 0, 0, 1636, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 61, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 1, 1, 8103, 0, 297, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16382, 4654, 16376, 15494, 15016, 15996] dup_rows_per_dst=[4, 10936, 0, 140, 3272, 2, 0, 34] combine_rows_per_src=[16382, 16384, 16382, 4654, 16376, 15494, 15016, 15996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128344 payload_rows_remote=111960 dup_rows=2728 dup_rows_remote=0 dedup_ratio=0.020813 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6828, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1364, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[13656, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2728, 0] combine_rows_per_src=[13656, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119044 payload_rows_remote=102660 dup_rows=12028 dup_rows_remote=12028 dedup_ratio=0.091766 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7060, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1132, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[15090, 16384, 16384, 16384, 16384, 16384, 5650, 16384] dup_rows_per_dst=[10734, 0, 0, 0, 0, 0, 1294, 0] combine_rows_per_src=[15090, 16384, 16384, 16384, 16384, 16384, 5650, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128830 payload_rows_remote=112446 dup_rows=2242 dup_rows_remote=2242 dedup_ratio=0.017105 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7071, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1121, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[14142, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2242, 0] combine_rows_per_src=[14142, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127352 payload_rows_remote=110968 dup_rows=3720 dup_rows_remote=3720 dedup_ratio=0.028381 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6332, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1860, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[12664, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 3720, 0] combine_rows_per_src=[12664, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=821673984 dispatch_meta_bytes=458744 combine_payload_bytes=821673984 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=822132728 ingress_bytes=972668928 link_bytes=972668928 token_bytes=1643806712 weight_movement_bytes=150994944 total_nvlink_bytes=1794801656 weight_to_token_ratio=0.091857 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.233481114e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=917176320 dispatch_meta_bytes=447840 combine_payload_bytes=917176320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=917624160 ingress_bytes=992673792 link_bytes=992673792 token_bytes=1834800480 weight_movement_bytes=75497472 total_nvlink_bytes=1910297952 weight_to_token_ratio=0.041148 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=43392 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.258850098e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114062 payload_rows_remote=97722 dup_rows=17010 dup_rows_remote=16624 dedup_ratio=0.129776 router_tokens_per_expert=[0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 1, 0, 59, 8, 3, 0, 8187, 0, 8180, 2, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 4, 0, 2, 0, 156, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 8031, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 1, 0, 0, 0, 3, 0, 1, 0, 2, 0, 0, 8129, 0, 0, 0, 8187, 0, 0, 0, 4, 3, 0, 0, 0, 0, 8166, 0, 0] payload_rows_per_dst=[78, 16368, 16372, 16040, 16100, 16380, 16384, 16340] dup_rows_per_dst=[0, 10, 16, 314, 0, 18, 16266, 386] combine_rows_per_src=[78, 16368, 16372, 16040, 16100, 16380, 16384, 16340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=840990720 dispatch_meta_bytes=458752 combine_payload_bytes=840990720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=841449472 ingress_bytes=916488192 link_bytes=916488192 token_bytes=1682440192 weight_movement_bytes=75497472 total_nvlink_bytes=1757937664 weight_to_token_ratio=0.044874 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.162236033e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126650 payload_rows_remote=110266 dup_rows=4422 dup_rows_remote=4422 dedup_ratio=0.033737 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5981, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2211, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[11962, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 4422, 0] combine_rows_per_src=[11962, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130906 payload_rows_remote=114570 dup_rows=166 dup_rows_remote=166 dedup_ratio=0.001266 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8168, 0, 0, 0, 0, 0, 76, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8133, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16336, 16384, 16384, 16384, 16384, 16266, 16384] dup_rows_per_dst=[12, 0, 0, 0, 152, 0, 0, 2] combine_rows_per_src=[16384, 16336, 16384, 16384, 16384, 16384, 16266, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=921157632 dispatch_meta_bytes=458752 combine_payload_bytes=921157632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=921616384 ingress_bytes=996655104 link_bytes=996655104 token_bytes=1842774016 weight_movement_bytes=75497472 total_nvlink_bytes=1918271488 weight_to_token_ratio=0.040969 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.263898962e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=909049856 dispatch_meta_bytes=458752 combine_payload_bytes=909049856 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=909508608 ingress_bytes=984547328 link_bytes=984547328 token_bytes=1818558464 weight_movement_bytes=75497472 total_nvlink_bytes=1894055936 weight_to_token_ratio=0.041515 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.248544597e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=800538624 dispatch_meta_bytes=457384 combine_payload_bytes=800538624 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=800996008 ingress_bytes=1027031040 link_bytes=1027031040 token_bytes=1601534632 weight_movement_bytes=226492416 total_nvlink_bytes=1828027048 weight_to_token_ratio=0.141422 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.302419924e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=903299072 dispatch_meta_bytes=458752 combine_payload_bytes=903299072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=903757824 ingress_bytes=978796544 link_bytes=978796544 token_bytes=1807056896 weight_movement_bytes=75497472 total_nvlink_bytes=1882554368 weight_to_token_ratio=0.041779 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.241251794e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938557440 dispatch_meta_bytes=458944 combine_payload_bytes=938557440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939016384 ingress_bytes=1014054912 link_bytes=1014054912 token_bytes=1877573824 weight_movement_bytes=75497472 total_nvlink_bytes=1953071296 weight_to_token_ratio=0.040210 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=52096 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285964367e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126870 payload_rows_remote=110488 dup_rows=4202 dup_rows_remote=340 dedup_ratio=0.032059 router_tokens_per_expert=[0, 0, 0, 0, 4, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1925, 1, 0, 0, 8191, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 1, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6279, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8179, 0, 0] payload_rows_per_dst=[16382, 16384, 16380, 16382, 16372, 16370, 12560, 16040] dup_rows_per_dst=[3862, 6, 0, 32, 296, 0, 4, 2] combine_rows_per_src=[16382, 16384, 16380, 16382, 16372, 16370, 12560, 16040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131068 payload_rows_remote=114684 dup_rows=4 dup_rows_remote=4 dedup_ratio=0.000031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 2, 0, 0] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131058 payload_rows_remote=114682 dup_rows=14 dup_rows_remote=14 dedup_ratio=0.000107 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16376, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 12, 0] combine_rows_per_src=[16384, 16378, 16376, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131072 payload_rows_remote=114688 dup_rows=0 dup_rows_remote=0 dedup_ratio=0.000000 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=905117696 dispatch_meta_bytes=443312 combine_payload_bytes=905117696 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=905561008 ingress_bytes=1131610112 link_bytes=1131610112 token_bytes=1810678704 weight_movement_bytes=226492416 total_nvlink_bytes=2037171120 weight_to_token_ratio=0.125087 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=43392 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.435040908e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939950080 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441408 weight_movement_bytes=75497472 total_nvlink_bytes=1954938880 weight_to_token_ratio=0.040170 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16384, 16382, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4, 2, 0, 0, 2, 0] combine_rows_per_src=[16384, 16378, 16384, 16382, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458784 combine_payload_bytes=939474944 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939933728 ingress_bytes=1090469888 link_bytes=1090469888 token_bytes=1879408672 weight_movement_bytes=150994944 total_nvlink_bytes=2030403616 weight_to_token_ratio=0.080342 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.053309 gemm_active_groups=7 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382869313e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=0 dedup_ratio=0.000015 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128706 payload_rows_remote=112324 dup_rows=2366 dup_rows_remote=2360 dedup_ratio=0.018051 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8187, 0, 0, 223, 8186, 0, 0, 15, 0, 226, 0, 0, 1, 0, 0, 0, 7478, 0, 0, 0, 0, 8184, 0, 7031, 0, 0, 0, 0, 1, 0, 8187, 0, 2, 0, 0, 1228, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3, 8176, 0, 6, 8187, 0, 0, 2, 1, 0, 0, 0, 0, 160, 0, 0, 1, 3, 20, 0, 0, 4, 0, 0, 0, 0, 4, 0, 4, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 15376, 15394, 16048, 16382, 16362, 16384, 16382] dup_rows_per_dst=[0, 450, 1046, 464, 4, 20, 376, 6] combine_rows_per_src=[16378, 15376, 15394, 16048, 16382, 16362, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458752 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939982848 ingress_bytes=1015021568 link_bytes=1015021568 token_bytes=1879506944 weight_movement_bytes=75497472 total_nvlink_bytes=1955004416 weight_to_token_ratio=0.040169 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287190223e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114954 payload_rows_remote=98570 dup_rows=16118 dup_rows_remote=16118 dedup_ratio=0.122971 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8059, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 133, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 266, 16384] dup_rows_per_dst=[16118, 0, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 266, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939917312 ingress_bytes=1090453504 link_bytes=1090453504 token_bytes=1879375872 weight_movement_bytes=150994944 total_nvlink_bytes=2030370816 weight_to_token_ratio=0.080343 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382848536e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458744 combine_payload_bytes=939507712 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=939966456 ingress_bytes=1241497600 link_bytes=1241497600 token_bytes=1879474168 weight_movement_bytes=301989888 total_nvlink_bytes=2181464056 weight_to_token_ratio=0.160678 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=63744 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.574393711e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=920158208 dispatch_meta_bytes=458736 combine_payload_bytes=920158208 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=920616944 ingress_bytes=1071153152 link_bytes=1071153152 token_bytes=1840775152 weight_movement_bytes=150994944 total_nvlink_bytes=1991770096 weight_to_token_ratio=0.082028 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.358372973e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=807485440 dispatch_meta_bytes=458752 combine_payload_bytes=807485440 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=807944192 ingress_bytes=1033977856 link_bytes=1033977856 token_bytes=1615429632 weight_movement_bytes=226492416 total_nvlink_bytes=1841922048 weight_to_token_ratio=0.140206 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.311229465e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121160 payload_rows_remote=104776 dup_rows=9912 dup_rows_remote=56 dedup_ratio=0.075623 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4928, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 6852, 16004, 16384, 16384, 16384] dup_rows_per_dst=[0, 56, 0, 0, 0, 0, 9856, 0] combine_rows_per_src=[16384, 16384, 16384, 6852, 16004, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130244 payload_rows_remote=113860 dup_rows=828 dup_rows_remote=822 dedup_ratio=0.006317 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 2, 0, 213, 8189, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 8189, 0, 7554, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 643, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 8, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16376, 16304, 16046, 16382, 16382, 15986, 16384] dup_rows_per_dst=[6, 582, 34, 198, 0, 2, 6, 0] combine_rows_per_src=[16384, 16376, 16304, 16046, 16382, 16382, 15986, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125280 payload_rows_remote=108896 dup_rows=5792 dup_rows_remote=5788 dedup_ratio=0.044189 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2931, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5296, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 10592, 16384] dup_rows_per_dst=[0, 56, 4, 5730, 2, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 10592, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118196 payload_rows_remote=102072 dup_rows=12876 dup_rows_remote=9024 dedup_ratio=0.098236 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3073, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5247, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 6146, 16124, 16384, 14006, 16384] dup_rows_per_dst=[0, 26, 0, 0, 3852, 0, 8998, 0] combine_rows_per_src=[16384, 16384, 16384, 6146, 16124, 16384, 14006, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=858324992 dispatch_meta_bytes=419328 combine_payload_bytes=858324992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=858744320 ingress_bytes=933822464 link_bytes=933822464 token_bytes=1717069312 weight_movement_bytes=75497472 total_nvlink_bytes=1792566784 weight_to_token_ratio=0.043969 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=34816 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.184218330e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123910 payload_rows_remote=107526 dup_rows=7162 dup_rows_remote=7162 dedup_ratio=0.054642 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 11, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3543, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 9478, 16128, 16384, 16384, 16384] dup_rows_per_dst=[0, 54, 0, 0, 22, 0, 7086, 0] combine_rows_per_src=[16384, 16384, 16384, 9478, 16128, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=892076032 dispatch_meta_bytes=458736 combine_payload_bytes=892076032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=892534768 ingress_bytes=967573504 link_bytes=967573504 token_bytes=1784610800 weight_movement_bytes=75497472 total_nvlink_bytes=1860108272 weight_to_token_ratio=0.042305 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.227019399e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122124 payload_rows_remote=105926 dup_rows=8948 dup_rows_remote=372 dedup_ratio=0.068268 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4375, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3811, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16198, 16384, 16384, 7622, 16384] dup_rows_per_dst=[0, 38, 0, 8576, 334, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16198, 16384, 16384, 7622, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=932741120 dispatch_meta_bytes=458728 combine_payload_bytes=932741120 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=933199848 ingress_bytes=1083736064 link_bytes=1083736064 token_bytes=1865940968 weight_movement_bytes=150994944 total_nvlink_bytes=2016935912 weight_to_token_ratio=0.080922 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.374329877e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=836173824 dispatch_meta_bytes=444384 combine_payload_bytes=836173824 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=836618208 ingress_bytes=1062666240 link_bytes=1062666240 token_bytes=1672792032 weight_movement_bytes=226492416 total_nvlink_bytes=1899284448 weight_to_token_ratio=0.135398 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=34816 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.347610374e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114682 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[0, 0, 0, 6, 0, 0, 2, 0] combine_rows_per_src=[16384, 16382, 16382, 16384, 16384, 16384, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122984 payload_rows_remote=106610 dup_rows=8088 dup_rows_remote=8076 dedup_ratio=0.061707 router_tokens_per_expert=[0, 0, 0, 0, 8182, 3, 0, 8180, 0, 0, 4, 0, 79, 0, 0, 0, 8187, 0, 616, 10, 0, 0, 1, 0, 0, 1, 0, 3, 1, 4, 0, 0, 8184, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 4, 0, 5269, 0, 0, 0, 1, 4, 0, 0, 0, 0, 8181, 0, 3, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2387, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[16206, 14044, 16380, 16058, 11174, 16372, 16376, 16374] dup_rows_per_dst=[148, 1262, 2, 10, 1876, 12, 4766, 12] combine_rows_per_src=[16206, 14044, 16380, 16058, 11174, 16372, 16376, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21128199312 prefetch_bytes=830472192 reduce_bytes=301989888 weight_movement_bytes=1132462080 egress_bytes=10566810768 ingress_bytes=11693850624 link_bytes=11693850624 total_nvlink_bytes=22260661392 weight_to_token_ratio=0.053600 prefetch_share_of_ingress=0.071018 dedup_ratio=0.056665 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=4 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051164 gemm_inner_pad_upper_bound=9525 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.235787385e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=880852992 dispatch_meta_bytes=458752 combine_payload_bytes=880852992 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=881311744 ingress_bytes=1031847936 link_bytes=1031847936 token_bytes=1762164736 weight_movement_bytes=150994944 total_nvlink_bytes=1913159680 weight_to_token_ratio=0.085687 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.308528426e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21526433752 prefetch_bytes=981467136 reduce_bytes=452984832 weight_movement_bytes=1434451968 egress_bytes=10765946840 ingress_bytes=12194938880 link_bytes=12194938880 total_nvlink_bytes=22960885720 weight_to_token_ratio=0.066637 prefetch_share_of_ingress=0.080482 dedup_ratio=0.040928 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=6 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.052083 gemm_inner_pad_upper_bound=9271 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.288741589e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[2, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21829744104 prefetch_bytes=754974720 reduce_bytes=1056964608 weight_movement_bytes=1811939328 egress_bytes=10917623272 ingress_bytes=12724060160 link_bytes=12724060160 total_nvlink_bytes=23641683432 weight_to_token_ratio=0.083003 prefetch_share_of_ingress=0.059334 dedup_ratio=0.033735 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=14 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.052390 gemm_inner_pad_upper_bound=8890 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.344658277e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=867745792 dispatch_meta_bytes=425192 combine_payload_bytes=867745792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=868170984 ingress_bytes=943243264 link_bytes=943243264 token_bytes=1735916776 weight_movement_bytes=75497472 total_nvlink_bytes=1811414248 weight_to_token_ratio=0.043491 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=35456 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.196165230e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939474944 dispatch_meta_bytes=458760 combine_payload_bytes=939474944 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939933704 ingress_bytes=1241464832 link_bytes=1241464832 token_bytes=1879408648 weight_movement_bytes=301989888 total_nvlink_bytes=2181398536 weight_to_token_ratio=0.160683 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.574352156e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21537666040 prefetch_bytes=1132462080 reduce_bytes=377487360 weight_movement_bytes=1509949440 egress_bytes=10771575800 ingress_bytes=12276039680 link_bytes=12276039680 total_nvlink_bytes=23047615480 weight_to_token_ratio=0.070107 prefetch_share_of_ingress=0.092250 dedup_ratio=0.039472 prefetch_slots=30 unused_prefetch_slots=354 slot_utilization=0.078125 reduce_slots=5 remote_demand=15 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050858 gemm_inner_pad_upper_bound=9779 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.297312191e-04 prefetch_bytes_per_gemm_flop=1.196767770e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=873349120 dispatch_meta_bytes=458744 combine_payload_bytes=873349120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=873807864 ingress_bytes=948846592 link_bytes=948846592 token_bytes=1747156984 weight_movement_bytes=75497472 total_nvlink_bytes=1822654456 weight_to_token_ratio=0.043212 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.203271039e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121748 payload_rows_remote=105364 dup_rows=9324 dup_rows_remote=9294 dedup_ratio=0.071136 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 8189, 0, 0, 1, 0, 17, 0, 0, 0, 8191, 0, 379, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5912, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3532, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16382, 16384, 16384, 7066, 16382] dup_rows_per_dst=[30, 764, 0, 8526, 4, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16382, 16384, 16384, 7066, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21631805544 prefetch_bytes=754974720 reduce_bytes=528482304 weight_movement_bytes=1283457024 egress_bytes=10818627688 ingress_bytes=12096634880 link_bytes=12097093632 total_nvlink_bytes=22915262568 weight_to_token_ratio=0.059332 prefetch_share_of_ingress=0.062412 dedup_ratio=0.035927 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=7 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051317 gemm_inner_pad_upper_bound=9525 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.278401460e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20753176032 prefetch_bytes=754974720 reduce_bytes=1358954496 weight_movement_bytes=2113929216 egress_bytes=10379302368 ingress_bytes=12487802880 link_bytes=12487802880 total_nvlink_bytes=22867105248 weight_to_token_ratio=0.101861 prefetch_share_of_ingress=0.060457 dedup_ratio=0.071640 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=18 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051164 gemm_inner_pad_upper_bound=9525 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.319690987e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=863141888 dispatch_meta_bytes=458632 combine_payload_bytes=863141888 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=863600520 ingress_bytes=1240629248 link_bytes=1240629248 token_bytes=1726742408 weight_movement_bytes=377487360 total_nvlink_bytes=2104229768 weight_to_token_ratio=0.218612 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.573292518e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126624 payload_rows_remote=110240 dup_rows=4448 dup_rows_remote=4364 dedup_ratio=0.033936 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5968, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 11936, 16384] dup_rows_per_dst=[0, 84, 0, 4348, 16, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 11936, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19902824528 prefetch_bytes=905969664 reduce_bytes=1811939328 weight_movement_bytes=2717908992 egress_bytes=9954099280 ingress_bytes=12666634240 link_bytes=12666634240 total_nvlink_bytes=22620733520 weight_to_token_ratio=0.136559 prefetch_share_of_ingress=0.071524 dedup_ratio=0.103196 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=24 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050551 gemm_inner_pad_upper_bound=10033 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.338589598e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=903086080 dispatch_meta_bytes=458416 combine_payload_bytes=903086080 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=903544496 ingress_bytes=1054081024 link_bytes=1054081024 token_bytes=1806630576 weight_movement_bytes=150994944 total_nvlink_bytes=1957625520 weight_to_token_ratio=0.083578 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.336723112e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21012721408 prefetch_bytes=830472192 reduce_bytes=1056964608 weight_movement_bytes=1887436800 egress_bytes=10509020928 ingress_bytes=12391137280 link_bytes=12391137280 total_nvlink_bytes=22900158208 weight_to_token_ratio=0.089824 prefetch_share_of_ingress=0.067021 dedup_ratio=0.059837 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=14 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049786 gemm_inner_pad_upper_bound=11430 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.309475521e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none + [2026-08-29 23:35:12] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 368.0 | throughput per GPU (TFLOP/s/GPU): 491.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.807828E+00 | loss scale: 1.0 | grad norm: 5.545 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=96708 payload_rows_remote=81106 dup_rows=34364 dup_rows_remote=24566 dedup_ratio=0.262177 router_tokens_per_expert=[1, 0, 0, 0, 0, 15, 0, 1, 0, 6570, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 2421, 0, 0, 985, 1956, 0, 0, 8186, 9, 1198, 0, 0, 0, 0, 3, 0, 0, 22, 0, 0, 0, 0, 0, 2702, 0, 0, 8, 7453, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2475, 0, 0, 0, 0, 8191, 0, 0, 0, 585, 0, 0, 0, 2196, 0, 2512, 5365, 0, 0, 342, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 2079, 0, 0, 0, 0, 242, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4958, 65, 0, 12, 1, 0, 0, 0, 0, 1, 0, 1, 4971, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[13154, 14724, 15602, 7506, 9846, 11720, 14210, 9946] dup_rows_per_dst=[26, 11102, 9798, 1128, 3090, 2626, 6592, 2] combine_rows_per_src=[13154, 14724, 15602, 7506, 9846, 11720, 14210, 9946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99646 payload_rows_remote=84006 dup_rows=31426 dup_rows_remote=21804 dedup_ratio=0.239761 router_tokens_per_expert=[0, 0, 0, 0, 0, 50, 0, 0, 0, 6356, 0, 0, 0, 0, 0, 1, 0, 2, 0, 1, 2382, 0, 0, 1271, 2566, 0, 0, 8192, 21, 651, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 3009, 0, 0, 0, 7205, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2933, 0, 0, 0, 0, 8192, 0, 0, 0, 657, 0, 0, 0, 1936, 0, 2251, 4823, 0, 0, 369, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2817, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4232, 2, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 5415, 0, 0, 0, 0, 6, 0, 0, 0, 0] payload_rows_per_dst=[12734, 8906, 6022, 16384, 10218, 16384, 13358, 15640] dup_rows_per_dst=[80, 2720, 0, 5866, 2370, 6012, 4756, 9622] combine_rows_per_src=[12734, 8906, 6022, 16384, 10218, 16384, 13358, 15640] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100914 payload_rows_remote=86994 dup_rows=30158 dup_rows_remote=24486 dedup_ratio=0.230087 router_tokens_per_expert=[0, 0, 0, 0, 1, 8, 0, 0, 0, 6382, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1897, 0, 0, 1278, 3419, 0, 0, 8191, 15, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 2569, 0, 0, 3, 7776, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3061, 0, 0, 0, 0, 8191, 0, 0, 0, 407, 0, 0, 0, 1823, 0, 2456, 5443, 0, 0, 427, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1562, 0, 0, 0, 0, 70, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4293, 58, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 4846, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[12768, 11596, 5144, 16382, 8700, 16384, 13920, 16020] dup_rows_per_dst=[14, 4320, 6, 6122, 1526, 3264, 5672, 9234] combine_rows_per_src=[12768, 11596, 5144, 16382, 8700, 16384, 13920, 16020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102154 payload_rows_remote=85772 dup_rows=28918 dup_rows_remote=23346 dedup_ratio=0.220627 router_tokens_per_expert=[0, 0, 0, 0, 0, 28, 0, 0, 0, 6278, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2286, 0, 0, 1158, 2075, 0, 0, 8191, 36, 1190, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2862, 0, 0, 8, 7451, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2786, 0, 0, 0, 0, 8186, 0, 0, 0, 569, 0, 0, 0, 2542, 0, 3126, 4887, 1, 0, 229, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2130, 0, 0, 0, 0, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4420, 11, 0, 2, 0, 0, 0, 13, 0, 1, 0, 1, 4576, 0, 0, 0, 0, 0, 0, 1, 1, 0] payload_rows_per_dst=[12582, 10438, 15672, 16382, 8084, 16378, 13438, 9180] dup_rows_per_dst=[32, 3054, 8610, 5572, 1214, 5228, 5202, 6] combine_rows_per_src=[12582, 10438, 15672, 16382, 8084, 16378, 13438, 9180] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100288 payload_rows_remote=83910 dup_rows=30784 dup_rows_remote=24898 dedup_ratio=0.234863 router_tokens_per_expert=[0, 0, 0, 2, 2, 6, 0, 0, 0, 6862, 0, 0, 0, 0, 1, 0, 0, 8, 0, 1, 2138, 0, 0, 1316, 2594, 0, 0, 8191, 19, 1825, 0, 0, 1, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 2380, 0, 0, 4, 7892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3208, 0, 0, 0, 0, 8183, 0, 0, 0, 511, 0, 0, 0, 1685, 0, 2577, 4519, 0, 1, 277, 0, 0, 0, 0, 1, 0, 0, 0, 2, 0, 2747, 0, 0, 0, 2, 196, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3987, 22, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 4353, 1, 0, 0, 1, 8, 0, 0, 0, 0] payload_rows_per_dst=[13734, 11398, 4776, 16382, 8626, 16378, 12894, 16100] dup_rows_per_dst=[12, 4404, 6, 6418, 1476, 5886, 4172, 8410] combine_rows_per_src=[13734, 11398, 4776, 16382, 8626, 16378, 12894, 16100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101402 payload_rows_remote=91464 dup_rows=29670 dup_rows_remote=27602 dedup_ratio=0.226364 router_tokens_per_expert=[0, 0, 0, 0, 0, 36, 0, 0, 0, 6225, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2457, 1, 0, 1076, 2011, 1, 1, 8190, 6, 829, 8, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 3479, 0, 0, 0, 7536, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3129, 0, 1, 0, 0, 8189, 1, 0, 2, 780, 0, 0, 1, 2502, 0, 2361, 4799, 0, 0, 356, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2259, 0, 1, 0, 0, 97, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4632, 38, 0, 5, 0, 0, 0, 4, 1, 0, 0, 0, 4515, 0, 0, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[12502, 9872, 9288, 16384, 9938, 16384, 13560, 13474] dup_rows_per_dst=[22, 2908, 2192, 6260, 2068, 4710, 5388, 6122] combine_rows_per_src=[12502, 9872, 9288, 16384, 9938, 16384, 13560, 13474] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=664420352 dispatch_meta_bytes=422688 combine_payload_bytes=664420352 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=664843040 ingress_bytes=815415296 link_bytes=815415296 token_bytes=1329263392 weight_movement_bytes=150994944 total_nvlink_bytes=1480258336 weight_to_token_ratio=0.113593 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=32512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.034061374e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=712654848 dispatch_meta_bytes=445920 combine_payload_bytes=712654848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=713100768 ingress_bytes=788152320 link_bytes=788152320 token_bytes=1425755616 weight_movement_bytes=75497472 total_nvlink_bytes=1501253088 weight_to_token_ratio=0.052953 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=35200 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.994880826e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=688177152 dispatch_meta_bytes=423240 combine_payload_bytes=688177152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=688600392 ingress_bytes=763674624 link_bytes=763674624 token_bytes=1376777544 weight_movement_bytes=75497472 total_nvlink_bytes=1452275016 weight_to_token_ratio=0.054836 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=37376 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.684469186e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=702644224 dispatch_meta_bytes=436472 combine_payload_bytes=702644224 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=703080696 ingress_bytes=778141696 link_bytes=778141696 token_bytes=1405724920 weight_movement_bytes=75497472 total_nvlink_bytes=1481222392 weight_to_token_ratio=0.053707 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=42880 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.867932023e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=86046 payload_rows_remote=69662 dup_rows=45026 dup_rows_remote=32132 dedup_ratio=0.343521 router_tokens_per_expert=[1, 0, 0, 0, 0, 45, 0, 0, 0, 6399, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 2011, 0, 0, 1461, 1950, 0, 0, 8191, 28, 1426, 1, 0, 0, 0, 3, 0, 0, 6, 0, 0, 0, 0, 0, 3510, 0, 0, 1, 7723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2725, 0, 0, 0, 2, 8181, 0, 0, 0, 658, 0, 0, 0, 2016, 0, 2841, 5169, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2171, 0, 0, 0, 0, 227, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4176, 10, 0, 1, 0, 0, 0, 8, 0, 0, 0, 1, 4188, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 10290, 16058, 5454, 16384, 4710, 8374, 8392] dup_rows_per_dst=[12894, 3466, 12110, 0, 16464, 90, 0, 2] combine_rows_per_src=[16384, 10290, 16058, 5454, 16384, 4710, 8374, 8392] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=687390720 dispatch_meta_bytes=435232 combine_payload_bytes=687390720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=687825952 ingress_bytes=762888192 link_bytes=762888192 token_bytes=1375216672 weight_movement_bytes=75497472 total_nvlink_bytes=1450714144 weight_to_token_ratio=0.054899 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=45824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.674496121e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=749273088 dispatch_meta_bytes=476264 combine_payload_bytes=749273088 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=749749352 ingress_bytes=1051262976 link_bytes=1051262976 token_bytes=1499022440 weight_movement_bytes=301989888 total_nvlink_bytes=1801012328 weight_to_token_ratio=0.201458 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=19840 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.333149430e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=570671104 dispatch_meta_bytes=407176 combine_payload_bytes=570671104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=571078280 ingress_bytes=646168576 link_bytes=646168576 token_bytes=1141749384 weight_movement_bytes=75497472 total_nvlink_bytes=1217246856 weight_to_token_ratio=0.066124 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=51840 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.194327095e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127826 payload_rows_remote=112072 dup_rows=3246 dup_rows_remote=3082 dedup_ratio=0.024765 router_tokens_per_expert=[8191, 0, 0, 1, 0, 0, 8186, 0, 1520, 0, 0, 8191, 0, 0, 1, 0, 0, 1, 0, 0, 0, 8191, 1, 0, 0, 13, 0, 1, 0, 0, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6884, 1, 0, 0, 0, 0, 0, 0, 7780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15754, 16384, 13770, 16382, 16384, 16384] dup_rows_per_dst=[3042, 30, 164, 0, 0, 0, 6, 4] combine_rows_per_src=[16384, 16384, 15754, 16384, 13770, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117544 payload_rows_remote=101160 dup_rows=13528 dup_rows_remote=13528 dedup_ratio=0.103210 router_tokens_per_expert=[8191, 0, 1, 1, 0, 0, 8191, 1, 1543, 0, 0, 8181, 0, 0, 0, 0, 1, 1, 0, 0, 1, 8191, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6736, 0, 0, 0, 1, 0, 0, 0, 8059, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3090, 16384, 16158, 16384, 16380, 16382, 16384, 16382] dup_rows_per_dst=[2, 12, 46, 2, 13456, 0, 0, 10] combine_rows_per_src=[3090, 16384, 16158, 16384, 16380, 16382, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=89088 payload_rows_remote=75310 dup_rows=41984 dup_rows_remote=33888 dedup_ratio=0.320312 router_tokens_per_expert=[0, 3, 0, 3, 0, 10, 0, 0, 0, 6759, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 2304, 0, 0, 1412, 2575, 0, 0, 8189, 17, 1722, 1, 2, 0, 3, 0, 0, 0, 0, 0, 2, 1, 0, 0, 2870, 0, 1, 0, 7614, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2415, 0, 0, 0, 0, 8188, 0, 1, 0, 700, 0, 0, 0, 1993, 0, 2403, 4805, 0, 0, 186, 0, 0, 1, 0, 0, 2, 0, 0, 3, 0, 2219, 0, 0, 0, 0, 234, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4424, 4, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 4451, 0, 0, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[15450, 13778, 15802, 4832, 16378, 4800, 9136, 8912] dup_rows_per_dst=[8684, 8096, 9986, 0, 14690, 118, 400, 10] combine_rows_per_src=[15450, 13778, 15802, 4832, 16378, 4800, 9136, 8912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118762 payload_rows_remote=102378 dup_rows=12310 dup_rows_remote=12306 dedup_ratio=0.093918 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 3035, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 641, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4537, 0, 0, 0, 0, 0, 0, 0, 8104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6072, 16384, 14386, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 128, 1114, 0, 11064, 0, 0, 4] combine_rows_per_src=[6072, 16384, 14386, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125190 payload_rows_remote=108808 dup_rows=5882 dup_rows_remote=5882 dedup_ratio=0.044876 router_tokens_per_expert=[8192, 0, 1, 0, 0, 0, 8192, 0, 2543, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5290, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16308, 16382, 10580, 16384, 16384, 16384] dup_rows_per_dst=[5086, 66, 706, 0, 0, 0, 2, 22] combine_rows_per_src=[16384, 16384, 16308, 16382, 10580, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128198 payload_rows_remote=114330 dup_rows=2874 dup_rows_remote=2872 dedup_ratio=0.021927 router_tokens_per_expert=[8190, 0, 0, 2, 0, 2, 8190, 0, 1396, 0, 0, 8187, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 104, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 8190, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6932, 2, 0, 0, 1, 0, 0, 0, 7928, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 16038, 16382, 13868, 16380, 16380, 16384] dup_rows_per_dst=[2794, 34, 28, 2, 2, 0, 8, 6] combine_rows_per_src=[16382, 16384, 16038, 16382, 13868, 16380, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121606 payload_rows_remote=105224 dup_rows=9466 dup_rows_remote=9466 dedup_ratio=0.072220 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 8188, 0, 3120, 0, 1, 8190, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5014, 0, 0, 0, 0, 0, 0, 0, 8007, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[11948, 16380, 16074, 16382, 11676, 16382, 16380, 16384] dup_rows_per_dst=[4584, 12, 392, 0, 4448, 0, 2, 28] combine_rows_per_src=[11948, 16380, 16074, 16382, 11676, 16382, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=828702720 dispatch_meta_bytes=458752 combine_payload_bytes=828702720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=829161472 ingress_bytes=904200192 link_bytes=904200192 token_bytes=1657864192 weight_movement_bytes=75497472 total_nvlink_bytes=1733361664 weight_to_token_ratio=0.045539 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.146653119e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=918093824 dispatch_meta_bytes=460616 combine_payload_bytes=918093824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=918554440 ingress_bytes=993591296 link_bytes=993591296 token_bytes=1836648264 weight_movement_bytes=75497472 total_nvlink_bytes=1912145736 weight_to_token_ratio=0.041106 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=62976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.260013622e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=838680576 dispatch_meta_bytes=458736 combine_payload_bytes=838680576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=839139312 ingress_bytes=914178048 link_bytes=914178048 token_bytes=1677819888 weight_movement_bytes=75497472 total_nvlink_bytes=1753317360 weight_to_token_ratio=0.044997 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.159306445e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=891355136 dispatch_meta_bytes=458760 combine_payload_bytes=891355136 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=891813896 ingress_bytes=1042350080 link_bytes=1042350080 token_bytes=1783169032 weight_movement_bytes=150994944 total_nvlink_bytes=1934163976 weight_to_token_ratio=0.084678 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.321846623e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125836 payload_rows_remote=109452 dup_rows=5236 dup_rows_remote=632 dedup_ratio=0.039948 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 2037, 0, 0, 8187, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5850, 0, 0, 0, 0, 0, 0, 0, 8148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15924, 16372, 16384, 11700, 16384, 16382, 16306] dup_rows_per_dst=[4604, 2, 620, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 15924, 16372, 16384, 11700, 16384, 16382, 16306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=936591360 dispatch_meta_bytes=468808 combine_payload_bytes=936591360 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937060168 ingress_bytes=1087586304 link_bytes=1087586304 token_bytes=1873651528 weight_movement_bytes=150994944 total_nvlink_bytes=2024646472 weight_to_token_ratio=0.080589 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=45184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.379212523e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=861995008 dispatch_meta_bytes=458760 combine_payload_bytes=861995008 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=862453768 ingress_bytes=937492480 link_bytes=937492480 token_bytes=1724448776 weight_movement_bytes=75497472 total_nvlink_bytes=1799946248 weight_to_token_ratio=0.043781 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.188872427e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=616939520 dispatch_meta_bytes=436792 combine_payload_bytes=616939520 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=617376312 ingress_bytes=843431936 link_bytes=843431936 token_bytes=1234315832 weight_movement_bytes=226492416 total_nvlink_bytes=1460808248 weight_to_token_ratio=0.183496 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.038603 gemm_active_groups=15 gemm_peak_group_rows=18176 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.069590417e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=896630784 dispatch_meta_bytes=440336 combine_payload_bytes=896630784 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=897071120 ingress_bytes=1198620672 link_bytes=1198620672 token_bytes=1793701904 weight_movement_bytes=301989888 total_nvlink_bytes=2095691792 weight_to_token_ratio=0.168361 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.042279 gemm_active_groups=12 gemm_peak_group_rows=46080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.520019731e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128296 payload_rows_remote=111912 dup_rows=2776 dup_rows_remote=2776 dedup_ratio=0.021179 router_tokens_per_expert=[0, 0, 2, 0, 0, 0, 0, 6818, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 8191, 0, 0, 0, 0, 1385, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8189, 0, 0, 1, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[13640, 16384, 16382, 16358, 16384, 16384, 16384, 16380] dup_rows_per_dst=[0, 2, 2770, 0, 2, 0, 0, 2] combine_rows_per_src=[13640, 16384, 16382, 16358, 16384, 16384, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125994 payload_rows_remote=109612 dup_rows=5078 dup_rows_remote=4980 dedup_ratio=0.038742 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 6355, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8045, 0, 0, 0, 0, 2062, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2] payload_rows_per_dst=[12710, 16154, 15456, 16228, 16374, 16384, 16306, 16382] dup_rows_per_dst=[0, 0, 3722, 1252, 0, 2, 4, 98] combine_rows_per_src=[12710, 16154, 15456, 16228, 16374, 16384, 16306, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124390 payload_rows_remote=108006 dup_rows=6682 dup_rows_remote=4 dedup_ratio=0.050980 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 4853, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8187, 0, 0, 0, 0, 3343, 0, 0, 0, 0, 0, 0, 8191, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[9710, 16384, 16384, 16378, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 6678, 0, 2, 2, 0, 0] combine_rows_per_src=[9710, 16384, 16384, 16378, 16384, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126018 payload_rows_remote=109634 dup_rows=5054 dup_rows_remote=5050 dedup_ratio=0.038559 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5675, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 8190, 0, 0, 0, 2, 2519, 0, 0, 0, 0, 1, 0, 8190, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8189, 0, 0, 2, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0] payload_rows_per_dst=[11350, 16384, 16384, 16370, 16384, 16380, 16384, 16382] dup_rows_per_dst=[0, 0, 5046, 0, 4, 0, 0, 4] combine_rows_per_src=[11350, 16384, 16384, 16370, 16384, 16380, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130504 payload_rows_remote=114120 dup_rows=568 dup_rows_remote=566 dedup_ratio=0.004333 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8168, 0, 0, 0, 0, 280, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[15846, 16384, 16358, 16382, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 26, 538, 0, 0, 2, 2, 0] combine_rows_per_src=[15846, 16384, 16358, 16382, 16384, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128570 payload_rows_remote=112366 dup_rows=2502 dup_rows_remote=2502 dedup_ratio=0.019089 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 7066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8107, 0, 0, 0, 0, 1265, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 8186, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[14132, 16384, 16326, 16204, 16384, 16384, 16384, 16372] dup_rows_per_dst=[0, 62, 2418, 0, 0, 0, 22, 0] combine_rows_per_src=[14132, 16384, 16326, 16204, 16384, 16384, 16384, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=916783104 dispatch_meta_bytes=458752 combine_payload_bytes=916783104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=917241856 ingress_bytes=992280576 link_bytes=992280576 token_bytes=1834024960 weight_movement_bytes=75497472 total_nvlink_bytes=1909522432 weight_to_token_ratio=0.041165 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.258351444e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122116 payload_rows_remote=105732 dup_rows=8956 dup_rows_remote=8776 dedup_ratio=0.068329 router_tokens_per_expert=[8189, 0, 10, 0, 0, 0, 8185, 0, 3964, 0, 0, 8182, 0, 0, 0, 2, 0, 0, 0, 0, 24, 8190, 0, 0, 0, 67, 0, 0, 0, 0, 1, 0, 0, 0, 727, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 3872, 0, 0, 0, 0, 0, 0, 0, 7729, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16378, 16384, 16098, 16374, 7744, 16384, 16374, 16380] dup_rows_per_dst=[7938, 180, 814, 2, 0, 10, 2, 10] combine_rows_per_src=[16378, 16384, 16098, 16374, 7744, 16384, 16374, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=897941504 dispatch_meta_bytes=458368 combine_payload_bytes=897941504 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=898399872 ingress_bytes=1124433920 link_bytes=1124433920 token_bytes=1796341376 weight_movement_bytes=226492416 total_nvlink_bytes=2022833792 weight_to_token_ratio=0.126085 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.425940487e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=884785152 dispatch_meta_bytes=432040 combine_payload_bytes=884785152 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=885217192 ingress_bytes=1186775040 link_bytes=1186775040 token_bytes=1770002344 weight_movement_bytes=301989888 total_nvlink_bytes=2071992232 weight_to_token_ratio=0.170616 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=48640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.504997802e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=898121728 dispatch_meta_bytes=458736 combine_payload_bytes=898121728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=898580464 ingress_bytes=973619200 link_bytes=973619200 token_bytes=1796702192 weight_movement_bytes=75497472 total_nvlink_bytes=1872199664 weight_to_token_ratio=0.042020 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.234686193e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120068 payload_rows_remote=103684 dup_rows=11004 dup_rows_remote=62 dedup_ratio=0.083954 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5712, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7951, 0, 0, 0, 0, 2944, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7942, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 8189, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 5904, 15884, 16384, 16366, 16384, 16378] dup_rows_per_dst=[10942, 48, 2, 0, 0, 0, 12, 0] combine_rows_per_src=[16384, 16384, 5904, 15884, 16384, 16366, 16384, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=934871040 dispatch_meta_bytes=458744 combine_payload_bytes=934871040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935329784 ingress_bytes=1010368512 link_bytes=1010368512 token_bytes=1870200824 weight_movement_bytes=75497472 total_nvlink_bytes=1945698296 weight_to_token_ratio=0.040369 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281289493e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=920502272 dispatch_meta_bytes=459472 combine_payload_bytes=920502272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=920961744 ingress_bytes=995999744 link_bytes=995999744 token_bytes=1841464016 weight_movement_bytes=75497472 total_nvlink_bytes=1916961488 weight_to_token_ratio=0.040999 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.263067873e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=866156544 dispatch_meta_bytes=458032 combine_payload_bytes=866156544 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=866614576 ingress_bytes=1017151488 link_bytes=1017151488 token_bytes=1732771120 weight_movement_bytes=150994944 total_nvlink_bytes=1883766064 weight_to_token_ratio=0.087141 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.289891262e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=849379328 dispatch_meta_bytes=414984 combine_payload_bytes=849379328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=849794312 ingress_bytes=924876800 link_bytes=924876800 token_bytes=1699173640 weight_movement_bytes=75497472 total_nvlink_bytes=1774671112 weight_to_token_ratio=0.044432 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=50176 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.172873969e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126590 payload_rows_remote=110208 dup_rows=4482 dup_rows_remote=26 dedup_ratio=0.034195 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5966, 0, 1, 1, 0, 0, 0, 8191, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2228, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 1, 0, 8189, 8191, 0, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16382, 11934, 16362, 16382, 16380] dup_rows_per_dst=[0, 18, 2, 2, 2, 0, 4456, 2] combine_rows_per_src=[16382, 16384, 16384, 16382, 11934, 16362, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130662 payload_rows_remote=114278 dup_rows=410 dup_rows_remote=32 dedup_ratio=0.003128 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16378, 16326, 16070, 16384, 16360, 16376, 16384, 16384] dup_rows_per_dst=[0, 0, 28, 0, 0, 0, 4, 378] combine_rows_per_src=[16378, 16326, 16070, 16384, 16360, 16376, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126254 payload_rows_remote=109870 dup_rows=4818 dup_rows_remote=4816 dedup_ratio=0.036758 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5793, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2406, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 1, 0, 8190, 8191, 0, 0, 1] payload_rows_per_dst=[16382, 16384, 16384, 16384, 11588, 16366, 16384, 16382] dup_rows_per_dst=[0, 2, 2, 0, 0, 0, 4812, 2] combine_rows_per_src=[16382, 16384, 16384, 16384, 11588, 16366, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124702 payload_rows_remote=114630 dup_rows=6370 dup_rows_remote=6370 dedup_ratio=0.048599 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5034, 0, 2, 0, 0, 0, 0, 8189, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3176, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 2, 0, 8188, 8190, 0, 0, 2] payload_rows_per_dst=[16380, 16384, 16384, 16382, 10072, 16336, 16384, 16380] dup_rows_per_dst=[0, 12, 4, 0, 0, 0, 6350, 4] combine_rows_per_src=[16380, 16384, 16384, 16382, 10072, 16336, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131044 payload_rows_remote=114672 dup_rows=28 dup_rows_remote=28 dedup_ratio=0.000214 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16368, 16372, 16384, 16384] dup_rows_per_dst=[2, 0, 12, 0, 0, 0, 14, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16368, 16372, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131024 payload_rows_remote=114640 dup_rows=48 dup_rows_remote=48 dedup_ratio=0.000366 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16344, 16376, 16384, 16384] dup_rows_per_dst=[2, 0, 40, 0, 0, 0, 6, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16344, 16376, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=902823936 dispatch_meta_bytes=440936 combine_payload_bytes=902823936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=903264872 ingress_bytes=978321408 link_bytes=978321408 token_bytes=1806088808 weight_movement_bytes=75497472 total_nvlink_bytes=1881586280 weight_to_token_ratio=0.041802 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=57728 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.240649254e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120664 payload_rows_remote=104280 dup_rows=10408 dup_rows_remote=10316 dedup_ratio=0.079407 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 5685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7828, 0, 0, 0, 0, 3018, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8002, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 8154, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16244, 16384, 6560, 16016, 16384, 16384, 16384, 16308] dup_rows_per_dst=[10114, 92, 144, 0, 2, 18, 38, 0] combine_rows_per_src=[16244, 16384, 6560, 16016, 16384, 16384, 16384, 16308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115066 payload_rows_remote=98682 dup_rows=16006 dup_rows_remote=16006 dedup_ratio=0.122116 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 754, 16008] dup_rows_per_dst=[0, 0, 36, 2, 15826, 142, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 754, 16008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=936165376 dispatch_meta_bytes=457240 combine_payload_bytes=936165376 prefetch_bytes=0 reduce_bytes=452984832 egress_bytes=936622616 ingress_bytes=1389150208 link_bytes=1389150208 token_bytes=1872787992 weight_movement_bytes=452984832 total_nvlink_bytes=2325772824 weight_to_token_ratio=0.241877 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.761638001e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=900055040 dispatch_meta_bytes=458744 combine_payload_bytes=900055040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=900513784 ingress_bytes=975552512 link_bytes=975552512 token_bytes=1800568824 weight_movement_bytes=75497472 total_nvlink_bytes=1876066296 weight_to_token_ratio=0.041930 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.237137905e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939048960 dispatch_meta_bytes=484000 combine_payload_bytes=939048960 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939532960 ingress_bytes=1090043904 link_bytes=1090043904 token_bytes=1878581920 weight_movement_bytes=150994944 total_nvlink_bytes=2029576864 weight_to_token_ratio=0.080377 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=57600 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382329106e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939393024 dispatch_meta_bytes=458800 combine_payload_bytes=939393024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939851824 ingress_bytes=1014890496 link_bytes=1014890496 token_bytes=1879244848 weight_movement_bytes=75497472 total_nvlink_bytes=1954742320 weight_to_token_ratio=0.040174 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287024006e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939130880 dispatch_meta_bytes=458752 combine_payload_bytes=939130880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939589632 ingress_bytes=1014628352 link_bytes=1014628352 token_bytes=1878720512 weight_movement_bytes=75497472 total_nvlink_bytes=1954217984 weight_to_token_ratio=0.040186 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286691570e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=854261760 dispatch_meta_bytes=458384 combine_payload_bytes=854261760 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=854720144 ingress_bytes=1005256704 link_bytes=1005256704 token_bytes=1708981904 weight_movement_bytes=150994944 total_nvlink_bytes=1859976848 weight_to_token_ratio=0.088354 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.274807001e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=808402944 dispatch_meta_bytes=458752 combine_payload_bytes=808402944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=808861696 ingress_bytes=883900416 link_bytes=883900416 token_bytes=1617264640 weight_movement_bytes=75497472 total_nvlink_bytes=1692762112 weight_to_token_ratio=0.046682 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.120910146e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128958 payload_rows_remote=112574 dup_rows=2114 dup_rows_remote=2032 dedup_ratio=0.016129 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 8164, 971, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 7978, 0, 0, 0, 31, 1, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8191, 8127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 43, 0, 1, 0, 0, 0, 0, 7438, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16330, 15958, 16384, 16382, 16258, 16384, 16384, 14878] dup_rows_per_dst=[1942, 2, 30, 4, 54, 0, 82, 0] combine_rows_per_src=[16330, 15958, 16384, 16382, 16258, 16384, 16384, 14878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128934 payload_rows_remote=112550 dup_rows=2138 dup_rows_remote=2036 dedup_ratio=0.016312 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 8174, 990, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 7905, 0, 0, 0, 50, 1, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 20, 0, 0, 8191, 8098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 10, 0, 1, 0, 0, 0, 0, 7506, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16362, 15812, 16384, 16382, 16214, 16384, 16382, 15014] dup_rows_per_dst=[1968, 0, 102, 26, 22, 0, 20, 0] combine_rows_per_src=[16362, 15812, 16384, 16382, 16214, 16384, 16382, 15014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123546 payload_rows_remote=107162 dup_rows=7526 dup_rows_remote=2848 dedup_ratio=0.057419 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6777, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1895, 0, 0, 8192, 7754, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[13540, 16292, 15960, 16384, 12218, 16384, 16384, 16384] dup_rows_per_dst=[2, 0, 0, 0, 2826, 20, 0, 4678] combine_rows_per_src=[13540, 16292, 15960, 16384, 12218, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128446 payload_rows_remote=112482 dup_rows=2626 dup_rows_remote=2626 dedup_ratio=0.020035 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8181, 0, 0, 2, 0, 0, 0, 0, 8168, 1174, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 7991, 0, 0, 0, 126, 2, 2, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 0, 0, 8190, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 33, 0, 2, 0, 0, 0, 0, 7290, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15986, 16384, 16380, 15964, 16384, 16382, 14582] dup_rows_per_dst=[2304, 2, 256, 6, 0, 6, 52, 0] combine_rows_per_src=[16384, 15986, 16384, 16380, 15964, 16384, 16382, 14582] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128838 payload_rows_remote=112458 dup_rows=2234 dup_rows_remote=2234 dedup_ratio=0.017044 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1483, 0, 0, 8174, 7813, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[14204, 16358, 16384, 16360, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 22, 4, 2208, 0, 0, 0] combine_rows_per_src=[14204, 16358, 16384, 16360, 16384, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130024 payload_rows_remote=113706 dup_rows=1048 dup_rows_remote=1044 dedup_ratio=0.007996 router_tokens_per_expert=[0, 0, 8189, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 7718, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1169, 0, 0, 8155, 7542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15442, 16350, 16384, 16318, 16384, 16378, 16384, 16384] dup_rows_per_dst=[0, 0, 6, 4, 1038, 0, 0, 0] combine_rows_per_src=[15442, 16350, 16384, 16318, 16384, 16378, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=922206208 dispatch_meta_bytes=458424 combine_payload_bytes=922206208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=922664632 ingress_bytes=997703680 link_bytes=997703680 token_bytes=1844870840 weight_movement_bytes=75497472 total_nvlink_bytes=1920368312 weight_to_token_ratio=0.040923 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265228704e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 2, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126698 payload_rows_remote=110486 dup_rows=4374 dup_rows_remote=1848 dedup_ratio=0.033371 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 7081, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1709, 0, 0, 8190, 7595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16212, 16378, 16384, 16380, 12342, 16384, 16234, 16384] dup_rows_per_dst=[2526, 1682, 2, 58, 106, 0, 0, 0] combine_rows_per_src=[16212, 16378, 16384, 16380, 12342, 16384, 16234, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130846 payload_rows_remote=114462 dup_rows=226 dup_rows_remote=200 dedup_ratio=0.001724 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16194, 16358, 16374, 16384] dup_rows_per_dst=[10, 26, 190, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16194, 16358, 16374, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=922009600 dispatch_meta_bytes=458344 combine_payload_bytes=922009600 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=922467944 ingress_bytes=997507072 link_bytes=997507072 token_bytes=1844477544 weight_movement_bytes=75497472 total_nvlink_bytes=1919975016 weight_to_token_ratio=0.040932 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.264979377e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=921452544 dispatch_meta_bytes=460432 combine_payload_bytes=921452544 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=921912976 ingress_bytes=1374437376 link_bytes=1374437376 token_bytes=1843365520 weight_movement_bytes=452984832 total_nvlink_bytes=2296350352 weight_to_token_ratio=0.245738 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.056985 gemm_active_groups=4 gemm_peak_group_rows=56960 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.742980059e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=921255936 dispatch_meta_bytes=458768 combine_payload_bytes=921255936 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=921714704 ingress_bytes=1072250880 link_bytes=1072250880 token_bytes=1842970640 weight_movement_bytes=150994944 total_nvlink_bytes=1993965584 weight_to_token_ratio=0.081930 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.359765047e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=877871104 dispatch_meta_bytes=440040 combine_payload_bytes=877871104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=878311144 ingress_bytes=953368576 link_bytes=953368576 token_bytes=1756182248 weight_movement_bytes=75497472 total_nvlink_bytes=1831679720 weight_to_token_ratio=0.042990 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=63232 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.209005551e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=931479552 dispatch_meta_bytes=459000 combine_payload_bytes=931479552 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=931938552 ingress_bytes=1082474496 link_bytes=1082474496 token_bytes=1863418104 weight_movement_bytes=150994944 total_nvlink_bytes=2014413048 weight_to_token_ratio=0.081031 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.372730031e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=905101312 dispatch_meta_bytes=449336 combine_payload_bytes=905101312 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=905550648 ingress_bytes=1207091200 link_bytes=1207091200 token_bytes=1810651960 weight_movement_bytes=301989888 total_nvlink_bytes=2112641848 weight_to_token_ratio=0.166785 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=60160 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.530761553e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=937672704 dispatch_meta_bytes=458648 combine_payload_bytes=937672704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938131352 ingress_bytes=1013170176 link_bytes=1013170176 token_bytes=1875804056 weight_movement_bytes=75497472 total_nvlink_bytes=1951301528 weight_to_token_ratio=0.040248 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284842398e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122302 payload_rows_remote=105920 dup_rows=8770 dup_rows_remote=1174 dedup_ratio=0.066910 router_tokens_per_expert=[0, 0, 8187, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 4573, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7418, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 465, 0, 0, 0, 117, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3798, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16382, 9148, 14860, 16384, 16380, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 8, 1164, 0, 7596, 2] combine_rows_per_src=[16382, 16382, 9148, 14860, 16384, 16380, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122336 payload_rows_remote=113348 dup_rows=8736 dup_rows_remote=8736 dedup_ratio=0.066650 router_tokens_per_expert=[0, 0, 8188, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 1, 0, 0, 4493, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7424, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 378, 0, 0, 0, 187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3781, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16370, 8988, 15064, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 0, 40, 1130, 0, 7562, 4] combine_rows_per_src=[16380, 16370, 8988, 15064, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119966 payload_rows_remote=103582 dup_rows=11106 dup_rows_remote=10372 dedup_ratio=0.084732 router_tokens_per_expert=[0, 0, 8185, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 2, 0, 0, 3801, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7035, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 167, 0, 0, 0, 200, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 5182, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16382, 7606, 14072, 16384, 16384, 16380, 16384] dup_rows_per_dst=[0, 4, 0, 0, 734, 0, 10364, 4] combine_rows_per_src=[16374, 16382, 7606, 14072, 16384, 16384, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130686 payload_rows_remote=114302 dup_rows=386 dup_rows_remote=386 dedup_ratio=0.002945 router_tokens_per_expert=[0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8010, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 1, 8186, 0, 0, 0, 0, 179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16028, 16384, 16384, 16384, 16376, 16364] dup_rows_per_dst=[12, 0, 6, 0, 368, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16028, 16384, 16384, 16384, 16376, 16364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130732 payload_rows_remote=114348 dup_rows=340 dup_rows_remote=286 dedup_ratio=0.002594 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16362, 16384, 16098, 16384, 16384, 16384, 16352, 16384] dup_rows_per_dst=[0, 0, 0, 0, 286, 0, 0, 54] combine_rows_per_src=[16362, 16384, 16098, 16384, 16384, 16384, 16352, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=867696640 dispatch_meta_bytes=428376 combine_payload_bytes=867696640 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=868125016 ingress_bytes=1320681472 link_bytes=1320681472 token_bytes=1735821656 weight_movement_bytes=452984832 total_nvlink_bytes=2188806488 weight_to_token_ratio=0.260963 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=52864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.674810006e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130758 payload_rows_remote=114374 dup_rows=314 dup_rows_remote=314 dedup_ratio=0.002396 router_tokens_per_expert=[0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8042, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16092, 16384, 16384, 16384, 16376, 16370] dup_rows_per_dst=[6, 0, 0, 0, 308, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16092, 16384, 16384, 16384, 16376, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109788 payload_rows_remote=93404 dup_rows=21284 dup_rows_remote=21284 dedup_ratio=0.162384 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 11484, 16384, 0, 16384] dup_rows_per_dst=[0, 20, 16118, 5108, 38, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 11484, 16384, 0, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=928546816 dispatch_meta_bytes=488336 combine_payload_bytes=928546816 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=929035152 ingress_bytes=1155039232 link_bytes=1155039232 token_bytes=1857581968 weight_movement_bytes=226492416 total_nvlink_bytes=2084074384 weight_to_token_ratio=0.121929 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=52992 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.464752330e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128388 payload_rows_remote=112136 dup_rows=2684 dup_rows_remote=2684 dedup_ratio=0.020477 router_tokens_per_expert=[0, 0, 8184, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 6921, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 2110, 0, 0, 8182, 7417, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[13854, 16252, 16384, 16378, 16384, 16368, 16384, 16384] dup_rows_per_dst=[2, 0, 6, 6, 2670, 0, 0, 0] combine_rows_per_src=[13854, 16252, 16384, 16378, 16384, 16368, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=848543744 dispatch_meta_bytes=455816 combine_payload_bytes=848543744 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=848999560 ingress_bytes=999538688 link_bytes=999538688 token_bytes=1697543304 weight_movement_bytes=150994944 total_nvlink_bytes=1848538248 weight_to_token_ratio=0.088949 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=62976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.267555752e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=936361984 dispatch_meta_bytes=458752 combine_payload_bytes=936361984 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=936820736 ingress_bytes=1087356928 link_bytes=1087356928 token_bytes=1873182720 weight_movement_bytes=150994944 total_nvlink_bytes=2024177664 weight_to_token_ratio=0.080609 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378921642e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=936738816 dispatch_meta_bytes=458536 combine_payload_bytes=936738816 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937197352 ingress_bytes=1012236288 link_bytes=1012236288 token_bytes=1873936168 weight_movement_bytes=75497472 total_nvlink_bytes=1949433640 weight_to_token_ratio=0.040288 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283658096e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=936951808 dispatch_meta_bytes=458752 combine_payload_bytes=936951808 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937410560 ingress_bytes=1012449280 link_bytes=1012449280 token_bytes=1874362368 weight_movement_bytes=75497472 total_nvlink_bytes=1949859840 weight_to_token_ratio=0.040279 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=62848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283928200e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=765165568 dispatch_meta_bytes=458752 combine_payload_bytes=765165568 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=765624320 ingress_bytes=840663040 link_bytes=840663040 token_bytes=1530789888 weight_movement_bytes=75497472 total_nvlink_bytes=1606287360 weight_to_token_ratio=0.049319 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.066079067e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=918618112 dispatch_meta_bytes=459280 combine_payload_bytes=918618112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919077392 ingress_bytes=994115584 link_bytes=994115584 token_bytes=1837695504 weight_movement_bytes=75497472 total_nvlink_bytes=1913192976 weight_to_token_ratio=0.041083 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.260678493e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127670 payload_rows_remote=111286 dup_rows=3402 dup_rows_remote=3402 dedup_ratio=0.025955 router_tokens_per_expert=[0, 0, 0, 0, 41, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 1, 0, 210, 0, 8191, 0, 4, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1410, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 22, 0, 6739, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 8014] payload_rows_per_dst=[16382, 16384, 16232, 13500, 16376, 16384, 16384, 16028] dup_rows_per_dst=[82, 472, 2820, 28, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16232, 13500, 16376, 16384, 16384, 16028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127622 payload_rows_remote=111350 dup_rows=3450 dup_rows_remote=220 dedup_ratio=0.026321 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 1, 5, 50, 0, 8191, 0, 2, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1615, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 49, 0, 6828, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8135, 0, 0, 0, 0, 7877] payload_rows_per_dst=[16382, 16384, 16272, 13680, 16382, 16384, 16384, 15754] dup_rows_per_dst=[0, 140, 3230, 80, 0, 0, 0, 0] combine_rows_per_src=[16382, 16384, 16272, 13680, 16382, 16384, 16384, 15754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128584 payload_rows_remote=112214 dup_rows=2488 dup_rows_remote=2488 dedup_ratio=0.018982 router_tokens_per_expert=[0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 2, 0, 93, 0, 8190, 0, 9, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1033, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 1, 0, 71, 0, 7112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8061, 0, 0, 0, 0, 8138] payload_rows_per_dst=[16380, 16384, 16130, 14276, 16370, 16384, 16384, 16276] dup_rows_per_dst=[84, 238, 2060, 104, 0, 0, 0, 2] combine_rows_per_src=[16380, 16384, 16130, 14276, 16370, 16384, 16384, 16276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130964 payload_rows_remote=114608 dup_rows=108 dup_rows_remote=108 dedup_ratio=0.000824 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 14, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 8165] payload_rows_per_dst=[16384, 16384, 16384, 16364, 16382, 16356, 16378, 16332] dup_rows_per_dst=[64, 0, 44, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16364, 16382, 16356, 16378, 16332] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=911654912 dispatch_meta_bytes=458752 combine_payload_bytes=911654912 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=912113664 ingress_bytes=987152384 link_bytes=987152384 token_bytes=1823768576 weight_movement_bytes=75497472 total_nvlink_bytes=1899266048 weight_to_token_ratio=0.041396 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.251848175e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120024 payload_rows_remote=103936 dup_rows=11048 dup_rows_remote=686 dedup_ratio=0.084290 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 226, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8044] payload_rows_per_dst=[15992, 15518, 7062, 16212, 16384, 16384, 16384, 16088] dup_rows_per_dst=[0, 16, 452, 0, 34, 156, 28, 10362] combine_rows_per_src=[15992, 15518, 7062, 16212, 16384, 16384, 16384, 16088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130894 payload_rows_remote=114570 dup_rows=178 dup_rows_remote=178 dedup_ratio=0.001358 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8155] payload_rows_per_dst=[16384, 16384, 16384, 16324, 16384, 16346, 16378, 16310] dup_rows_per_dst=[50, 0, 128, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16324, 16384, 16346, 16378, 16310] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122720 payload_rows_remote=106336 dup_rows=8352 dup_rows_remote=8284 dedup_ratio=0.063721 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8000] payload_rows_per_dst=[16384, 16384, 16384, 16368, 16384, 16346, 16384, 8086] dup_rows_per_dst=[68, 0, 486, 7798, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16368, 16384, 16346, 16384, 8086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121928 payload_rows_remote=105548 dup_rows=9144 dup_rows_remote=9144 dedup_ratio=0.069763 router_tokens_per_expert=[0, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 7908, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 21, 0, 0, 0, 6, 8185, 0, 0, 0, 0, 267, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16356, 16384, 16384, 16384, 7302, 16354] dup_rows_per_dst=[20, 0, 8248, 0, 872, 4, 0, 0] combine_rows_per_src=[16384, 16380, 16356, 16384, 16384, 16384, 7302, 16354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=912179200 dispatch_meta_bytes=446280 combine_payload_bytes=912179200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=912625480 ingress_bytes=987676672 link_bytes=987676672 token_bytes=1824804680 weight_movement_bytes=75497472 total_nvlink_bytes=1900302152 weight_to_token_ratio=0.041373 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=60672 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.252513046e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=919257088 dispatch_meta_bytes=458808 combine_payload_bytes=919257088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919715896 ingress_bytes=994754560 link_bytes=994754560 token_bytes=1838972984 weight_movement_bytes=75497472 total_nvlink_bytes=1914470456 weight_to_token_ratio=0.041054 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.261488804e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=851443712 dispatch_meta_bytes=418488 combine_payload_bytes=851443712 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=851862200 ingress_bytes=1077936128 link_bytes=1077936128 token_bytes=1703305912 weight_movement_bytes=226492416 total_nvlink_bytes=1929798328 weight_to_token_ratio=0.132972 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=60416 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.366974741e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=938868736 dispatch_meta_bytes=458864 combine_payload_bytes=938868736 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939327600 ingress_bytes=1014366208 link_bytes=1014366208 token_bytes=1878196336 weight_movement_bytes=75497472 total_nvlink_bytes=1953693808 weight_to_token_ratio=0.040197 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286359135e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=938557440 dispatch_meta_bytes=458992 combine_payload_bytes=938557440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939016432 ingress_bytes=1014054912 link_bytes=1014054912 token_bytes=1877573872 weight_movement_bytes=75497472 total_nvlink_bytes=1953071344 weight_to_token_ratio=0.040210 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=61440 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285964367e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=871104512 dispatch_meta_bytes=458480 combine_payload_bytes=871104512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=871562992 ingress_bytes=946601984 link_bytes=946601984 token_bytes=1742667504 weight_movement_bytes=75497472 total_nvlink_bytes=1818164976 weight_to_token_ratio=0.043323 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.056985 gemm_active_groups=4 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.200424560e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=864649216 dispatch_meta_bytes=458768 combine_payload_bytes=864649216 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=865107984 ingress_bytes=1015644160 link_bytes=1015644160 token_bytes=1729757200 weight_movement_bytes=150994944 total_nvlink_bytes=1880752144 weight_to_token_ratio=0.087293 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287979758e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131030 payload_rows_remote=114646 dup_rows=42 dup_rows_remote=40 dedup_ratio=0.000320 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8191, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 5593, 0, 0, 2600, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 8191, 0] payload_rows_per_dst=[16382, 16358, 16376, 16384, 16382, 16382, 16384, 16382] dup_rows_per_dst=[4, 0, 2, 8, 2, 6, 2, 18] combine_rows_per_src=[16382, 16358, 16376, 16384, 16382, 16382, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131014 payload_rows_remote=114638 dup_rows=58 dup_rows_remote=54 dedup_ratio=0.000443 router_tokens_per_expert=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, 7550, 0, 0, 643, 2, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 8186, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8190, 0] payload_rows_per_dst=[16380, 16346, 16384, 16384, 16376, 16380, 16384, 16380] dup_rows_per_dst=[0, 0, 4, 14, 4, 22, 0, 14] combine_rows_per_src=[16380, 16346, 16384, 16384, 16376, 16380, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114634 dup_rows=56 dup_rows_remote=52 dedup_ratio=0.000427 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 5917, 0, 0, 2291, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8188, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 8191, 0] payload_rows_per_dst=[16382, 16362, 16382, 16384, 16378, 16364, 16382, 16382] dup_rows_per_dst=[0, 0, 4, 38, 2, 2, 4, 6] combine_rows_per_src=[16382, 16362, 16382, 16384, 16378, 16364, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939180032 dispatch_meta_bytes=458744 combine_payload_bytes=939180032 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=939638776 ingress_bytes=1090174976 link_bytes=1090174976 token_bytes=1878818808 weight_movement_bytes=150994944 total_nvlink_bytes=2029813752 weight_to_token_ratio=0.080367 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382495323e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131060 payload_rows_remote=114676 dup_rows=12 dup_rows_remote=12 dedup_ratio=0.000092 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16372, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 12, 0] combine_rows_per_src=[16372, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131066 payload_rows_remote=114684 dup_rows=6 dup_rows_remote=4 dedup_ratio=0.000046 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16382, 16382, 16382] dup_rows_per_dst=[0, 4, 0, 0, 0, 2, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16382, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131070 payload_rows_remote=114686 dup_rows=2 dup_rows_remote=2 dedup_ratio=0.000015 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 2, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130400 payload_rows_remote=114016 dup_rows=672 dup_rows_remote=672 dedup_ratio=0.005127 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 0, 0, 0, 0, 0, 0, 283, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8192, 0, 0, 0, 0, 7913] payload_rows_per_dst=[16384, 16384, 16384, 16338, 16376, 16332, 16368, 15834] dup_rows_per_dst=[52, 0, 620, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16338, 16376, 16332, 16368, 15834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130980 payload_rows_remote=114596 dup_rows=92 dup_rows_remote=76 dedup_ratio=0.000702 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 16324, 16384, 16376, 16384, 16360] dup_rows_per_dst=[16, 64, 4, 0, 8, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16324, 16384, 16376, 16384, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939114496 dispatch_meta_bytes=458768 combine_payload_bytes=939114496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939573264 ingress_bytes=1014611968 link_bytes=1014611968 token_bytes=1878687760 weight_movement_bytes=75497472 total_nvlink_bytes=1954185232 weight_to_token_ratio=0.040186 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286670793e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939081728 dispatch_meta_bytes=458744 combine_payload_bytes=939081728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939540472 ingress_bytes=1014579200 link_bytes=1014579200 token_bytes=1878622200 weight_movement_bytes=75497472 total_nvlink_bytes=1954119672 weight_to_token_ratio=0.040188 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286629238e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939425792 dispatch_meta_bytes=458752 combine_payload_bytes=939425792 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=939884544 ingress_bytes=1165918208 link_bytes=1165918208 token_bytes=1879310336 weight_movement_bytes=226492416 total_nvlink_bytes=2105802752 weight_to_token_ratio=0.120519 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478548403e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939491328 dispatch_meta_bytes=458752 combine_payload_bytes=939491328 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=939950080 ingress_bytes=1014988800 link_bytes=1014988800 token_bytes=1879441408 weight_movement_bytes=75497472 total_nvlink_bytes=1954938880 weight_to_token_ratio=0.040170 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287148669e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=458752 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939966464 ingress_bytes=1090502656 link_bytes=1090502656 token_bytes=1879474176 weight_movement_bytes=150994944 total_nvlink_bytes=2030469120 weight_to_token_ratio=0.080339 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=46464 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382910868e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=934019072 dispatch_meta_bytes=458752 combine_payload_bytes=934019072 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=934477824 ingress_bytes=1311506432 link_bytes=1311506432 token_bytes=1868496896 weight_movement_bytes=377487360 total_nvlink_bytes=2245984256 weight_to_token_ratio=0.202027 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.663174764e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938770432 dispatch_meta_bytes=458688 combine_payload_bytes=938770432 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939229120 ingress_bytes=1165262848 link_bytes=1165262848 token_bytes=1877999552 weight_movement_bytes=226492416 total_nvlink_bytes=2104491968 weight_to_token_ratio=0.120603 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.477717314e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129734 payload_rows_remote=113350 dup_rows=1338 dup_rows_remote=1338 dedup_ratio=0.010208 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 19, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 8191, 0, 0, 0, 0, 0, 0, 636, 0, 0, 0, 0, 0, 8166, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 3, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7561, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16362, 16382, 16382, 16334, 16384, 16384, 15124] dup_rows_per_dst=[4, 40, 0, 1286, 2, 4, 0, 2] combine_rows_per_src=[16382, 16362, 16382, 16382, 16334, 16384, 16384, 15124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129000 payload_rows_remote=112708 dup_rows=2072 dup_rows_remote=2068 dedup_ratio=0.015808 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 4, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 37, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8190, 0, 0, 0, 0, 0, 0, 982, 0, 0, 0, 0, 0, 8144, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 10, 0, 0, 0, 8185, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 7206, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16380, 16380, 16292, 16384, 16384, 14416] dup_rows_per_dst=[8, 78, 0, 1968, 4, 10, 0, 4] combine_rows_per_src=[16380, 16384, 16380, 16380, 16292, 16384, 16384, 14416] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=928563200 dispatch_meta_bytes=458752 combine_payload_bytes=928563200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=929021952 ingress_bytes=1004060672 link_bytes=1004060672 token_bytes=1857585152 weight_movement_bytes=75497472 total_nvlink_bytes=1933082624 weight_to_token_ratio=0.040643 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=63616 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.273290264e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129898 payload_rows_remote=113516 dup_rows=1174 dup_rows_remote=1174 dedup_ratio=0.008957 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 13, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 8191, 0, 0, 0, 0, 0, 0, 557, 0, 0, 0, 0, 0, 8175, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 7627, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16382, 16382, 16382, 16352, 16384, 16378, 15256] dup_rows_per_dst=[2, 28, 0, 1124, 2, 16, 0, 2] combine_rows_per_src=[16382, 16382, 16382, 16382, 16352, 16384, 16378, 15256] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121524 payload_rows_remote=105140 dup_rows=9548 dup_rows_remote=5200 dedup_ratio=0.072845 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 518, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16270, 16360, 8000, 16384, 16384, 15378, 16384] dup_rows_per_dst=[0, 0, 0, 998, 162, 44, 3996, 4348] combine_rows_per_src=[16364, 16270, 16360, 8000, 16384, 16384, 15378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130710 payload_rows_remote=114326 dup_rows=362 dup_rows_remote=362 dedup_ratio=0.002762 router_tokens_per_expert=[0, 3, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 175, 0, 8184, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 16384, 16384, 16024, 16384] dup_rows_per_dst=[6, 0, 2, 352, 2, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 16384, 16384, 16024, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130600 payload_rows_remote=114216 dup_rows=472 dup_rows_remote=6 dedup_ratio=0.003601 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 230, 0, 8181, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7956, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 15912, 16384] dup_rows_per_dst=[4, 0, 2, 466, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 15912, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131050 payload_rows_remote=114666 dup_rows=22 dup_rows_remote=6 dedup_ratio=0.000168 router_tokens_per_expert=[0, 8186, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 1, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0] payload_rows_per_dst=[16384, 16384, 16380, 16384, 16384, 16382, 16372, 16380] dup_rows_per_dst=[0, 16, 0, 0, 0, 6, 0, 0] combine_rows_per_src=[16384, 16384, 16380, 16384, 16384, 16382, 16372, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=923303936 dispatch_meta_bytes=459104 combine_payload_bytes=923303936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=923763040 ingress_bytes=998801408 link_bytes=998801408 token_bytes=1847066976 weight_movement_bytes=75497472 total_nvlink_bytes=1922564448 weight_to_token_ratio=0.040874 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.266620777e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130008 payload_rows_remote=113624 dup_rows=1064 dup_rows_remote=1064 dedup_ratio=0.008118 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 495, 0, 8189, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7697, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16310, 15394, 16384] dup_rows_per_dst=[0, 0, 0, 1064, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16310, 15394, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=929923072 dispatch_meta_bytes=458760 combine_payload_bytes=929923072 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=930381832 ingress_bytes=1080918016 link_bytes=1080918016 token_bytes=1860304904 weight_movement_bytes=150994944 total_nvlink_bytes=2011299848 weight_to_token_ratio=0.081167 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.370756195e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=861306880 dispatch_meta_bytes=441360 combine_payload_bytes=861306880 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=861748240 ingress_bytes=1012301824 link_bytes=1012301824 token_bytes=1723055120 weight_movement_bytes=150994944 total_nvlink_bytes=1874050064 weight_to_token_ratio=0.087632 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=63360 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283741205e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=936558592 dispatch_meta_bytes=458752 combine_payload_bytes=936558592 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937017344 ingress_bytes=1087553536 link_bytes=1087553536 token_bytes=1873575936 weight_movement_bytes=150994944 total_nvlink_bytes=2024570880 weight_to_token_ratio=0.080592 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.379170968e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=935657472 dispatch_meta_bytes=456888 combine_payload_bytes=935657472 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=936114360 ingress_bytes=1237647360 link_bytes=1237647360 token_bytes=1871771832 weight_movement_bytes=301989888 total_nvlink_bytes=2173761720 weight_to_token_ratio=0.161339 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.053309 gemm_active_groups=7 gemm_peak_group_rows=61312 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.569511065e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939343872 dispatch_meta_bytes=458688 combine_payload_bytes=939343872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939802560 ingress_bytes=1014841344 link_bytes=1014841344 token_bytes=1879146432 weight_movement_bytes=75497472 total_nvlink_bytes=1954643904 weight_to_token_ratio=0.040176 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286961674e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111358 payload_rows_remote=100412 dup_rows=19714 dup_rows_remote=19712 dedup_ratio=0.150406 router_tokens_per_expert=[0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 8191, 0, 8187, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1673, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6089, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5472, 0, 0, 0, 8191, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8176, 0, 0] payload_rows_per_dst=[16382, 16384, 16376, 16382, 12180, 6324, 10946, 16384] dup_rows_per_dst=[0, 8, 0, 3350, 0, 0, 2, 16354] combine_rows_per_src=[16382, 16384, 16376, 16382, 12180, 6324, 10946, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=930807808 dispatch_meta_bytes=458752 combine_payload_bytes=930807808 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=931266560 ingress_bytes=1006305280 link_bytes=1006305280 token_bytes=1862074368 weight_movement_bytes=75497472 total_nvlink_bytes=1937571840 weight_to_token_ratio=0.040545 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.276136743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=822575104 dispatch_meta_bytes=480496 combine_payload_bytes=822575104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=823055600 ingress_bytes=898072576 link_bytes=898072576 token_bytes=1645630704 weight_movement_bytes=75497472 total_nvlink_bytes=1721128176 weight_to_token_ratio=0.045878 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=57344 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.138882440e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115382 payload_rows_remote=98998 dup_rows=15690 dup_rows_remote=2444 dedup_ratio=0.119705 router_tokens_per_expert=[0, 0, 0, 0, 3, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8190, 0, 8180, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 1210, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6626, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2789, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 5793, 0, 0, 0, 8187, 0, 0, 0, 2, 2, 0, 0, 2, 0, 8153, 0, 0] payload_rows_per_dst=[16380, 16384, 16370, 16378, 16384, 5582, 11590, 16314] dup_rows_per_dst=[4, 10, 0, 2420, 13246, 0, 6, 4] combine_rows_per_src=[16380, 16384, 16370, 16378, 16384, 5582, 11590, 16314] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110134 payload_rows_remote=93756 dup_rows=20938 dup_rows_remote=20938 dedup_ratio=0.159744 router_tokens_per_expert=[0, 0, 0, 0, 7, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8191, 0, 8187, 0, 1, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 1118, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6701, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5615, 0, 0, 0, 8189, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8179, 0, 0] payload_rows_per_dst=[16382, 16384, 16378, 16380, 16384, 10612, 11232, 6382] dup_rows_per_dst=[14, 18, 0, 2236, 13398, 5268, 2, 2] combine_rows_per_src=[16382, 16384, 16378, 16380, 16384, 10612, 11232, 6382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98296 payload_rows_remote=81912 dup_rows=32776 dup_rows_remote=16392 dedup_ratio=0.250061 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 275, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16356, 56, 16384, 16384, 550, 15820, 16362, 16384] dup_rows_per_dst=[2, 0, 28, 52, 0, 0, 16310, 16384] combine_rows_per_src=[16356, 56, 16384, 16384, 550, 15820, 16362, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120168 payload_rows_remote=104796 dup_rows=10904 dup_rows_remote=10904 dedup_ratio=0.083191 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 8192, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7685, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8136, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 6602, 15372, 16274, 16384] dup_rows_per_dst=[0, 36, 0, 84, 688, 0, 0, 10096] combine_rows_per_src=[16384, 16384, 16384, 16384, 6602, 15372, 16274, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129344 payload_rows_remote=112960 dup_rows=1728 dup_rows_remote=1532 dedup_ratio=0.013184 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 8192, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 0, 709, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7450, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8070, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 14900, 16140, 16384] dup_rows_per_dst=[0, 56, 0, 196, 1476, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 14900, 16140, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129904 payload_rows_remote=113526 dup_rows=1168 dup_rows_remote=1168 dedup_ratio=0.008911 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 574, 0, 8155, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7611, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16384, 16384, 15222, 16384] dup_rows_per_dst=[4, 0, 28, 1134, 2, 0, 0, 0] combine_rows_per_src=[16384, 16378, 16384, 16384, 16384, 16384, 15222, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=105548 payload_rows_remote=89164 dup_rows=25524 dup_rows_remote=25520 dedup_ratio=0.194733 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 135, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 40, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7769, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8069, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 7810, 16384, 16064, 16138, 0] dup_rows_per_dst=[4, 16160, 0, 0, 794, 8566, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 7810, 16384, 16064, 16138, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=810991616 dispatch_meta_bytes=405768 combine_payload_bytes=810991616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=811397384 ingress_bytes=886489088 link_bytes=886489088 token_bytes=1622389000 weight_movement_bytes=75497472 total_nvlink_bytes=1697886472 weight_to_token_ratio=0.046535 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=44160 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.124192946e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=768049152 dispatch_meta_bytes=458776 combine_payload_bytes=768049152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=768507928 ingress_bytes=843546624 link_bytes=843546624 token_bytes=1536557080 weight_movement_bytes=75497472 total_nvlink_bytes=1612054552 weight_to_token_ratio=0.049134 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.069735857e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=671023104 dispatch_meta_bytes=393216 combine_payload_bytes=671023104 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=671416320 ingress_bytes=897515520 link_bytes=897515520 token_bytes=1342439424 weight_movement_bytes=226492416 total_nvlink_bytes=1568931840 weight_to_token_ratio=0.168717 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=44160 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.138176014e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=858488832 dispatch_meta_bytes=462800 combine_payload_bytes=858488832 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=858951632 ingress_bytes=933986304 link_bytes=933986304 token_bytes=1717440464 weight_movement_bytes=75497472 total_nvlink_bytes=1792937936 weight_to_token_ratio=0.043959 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=47872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.184426102e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126950 payload_rows_remote=114082 dup_rows=4122 dup_rows_remote=4112 dedup_ratio=0.031448 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 1910, 8191, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 7894, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 8191, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 223, 0, 0, 1, 0, 6210, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 15788, 16382, 16384, 12868, 16380] dup_rows_per_dst=[0, 3820, 0, 0, 290, 2, 10, 0] combine_rows_per_src=[16384, 16382, 16382, 15788, 16382, 16384, 12868, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=925368320 dispatch_meta_bytes=457968 combine_payload_bytes=925368320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=925826288 ingress_bytes=1000865792 link_bytes=1000865792 token_bytes=1851194608 weight_movement_bytes=75497472 total_nvlink_bytes=1926692080 weight_to_token_ratio=0.040783 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=61184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.269238707e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=930004992 dispatch_meta_bytes=458776 combine_payload_bytes=930004992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=930463768 ingress_bytes=1005502464 link_bytes=1005502464 token_bytes=1860468760 weight_movement_bytes=75497472 total_nvlink_bytes=1935966232 weight_to_token_ratio=0.040580 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.275118660e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=730431488 dispatch_meta_bytes=458736 combine_payload_bytes=730431488 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=730890224 ingress_bytes=805928960 link_bytes=805928960 token_bytes=1461321712 weight_movement_bytes=75497472 total_nvlink_bytes=1536819184 weight_to_token_ratio=0.051664 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.022031364e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=934559744 dispatch_meta_bytes=472776 combine_payload_bytes=934559744 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935032520 ingress_bytes=1010057216 link_bytes=1010057216 token_bytes=1869592264 weight_movement_bytes=75497472 total_nvlink_bytes=1945089736 weight_to_token_ratio=0.040382 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=56832 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.280894726e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125938 payload_rows_remote=109558 dup_rows=5134 dup_rows_remote=5098 dedup_ratio=0.039169 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 1, 0, 2531, 8189, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 7917, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 8186, 8190, 0, 0, 2, 0, 0, 0, 0, 0, 8, 0, 0, 1, 1, 517, 0, 0, 2, 0, 5397, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16370, 15834, 16380, 16376, 11832, 16382] dup_rows_per_dst=[2, 5064, 0, 0, 36, 4, 28, 0] combine_rows_per_src=[16384, 16380, 16370, 15834, 16380, 16376, 11832, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127282 payload_rows_remote=110916 dup_rows=3790 dup_rows_remote=3790 dedup_ratio=0.028915 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 6, 0, 1773, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 7940, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 8189, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 353, 0, 0, 1, 0, 6209, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16366, 15880, 16382, 16380, 13126, 16382] dup_rows_per_dst=[0, 3558, 0, 6, 214, 2, 10, 0] combine_rows_per_src=[16384, 16382, 16366, 15880, 16382, 16380, 13126, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104046 payload_rows_remote=87662 dup_rows=27026 dup_rows_remote=27018 dedup_ratio=0.206192 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 401, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7495, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 710, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7778, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 806, 16384, 14990, 6338, 16384, 16378, 16384] dup_rows_per_dst=[0, 0, 34, 0, 258, 12198, 14528, 8] combine_rows_per_src=[16382, 806, 16384, 14990, 6338, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126770 payload_rows_remote=110386 dup_rows=4302 dup_rows_remote=4302 dedup_ratio=0.032822 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 872, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 6929, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1281, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7299, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 13868, 16384, 16384, 14598, 16384] dup_rows_per_dst=[0, 1744, 0, 0, 2558, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 13868, 16384, 16384, 14598, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128678 payload_rows_remote=113968 dup_rows=2394 dup_rows_remote=2394 dedup_ratio=0.018265 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 147, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5, 0, 0, 0, 7350, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 14710, 16384, 16384, 15664, 16384] dup_rows_per_dst=[0, 294, 0, 0, 2100, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 14710, 16384, 16384, 15664, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118440 payload_rows_remote=102056 dup_rows=12632 dup_rows_remote=12632 dedup_ratio=0.096375 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 83, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 574, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16004, 16384, 4190, 16326, 16384] dup_rows_per_dst=[0, 166, 0, 11318, 1148, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16004, 16384, 4190, 16326, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=897499136 dispatch_meta_bytes=458624 combine_payload_bytes=897499136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=897957760 ingress_bytes=972996608 link_bytes=972996608 token_bytes=1795456896 weight_movement_bytes=75497472 total_nvlink_bytes=1870954368 weight_to_token_ratio=0.042049 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.233896659e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114374 payload_rows_remote=97990 dup_rows=16698 dup_rows_remote=16658 dedup_ratio=0.127396 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8176, 0, 8189, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 127, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 63, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 8016, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 16070, 0] dup_rows_per_dst=[12, 40, 0, 248, 218, 16180, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 16070, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=908623872 dispatch_meta_bytes=458824 combine_payload_bytes=908623872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=909082696 ingress_bytes=984121344 link_bytes=984121344 token_bytes=1817706568 weight_movement_bytes=75497472 total_nvlink_bytes=1893204040 weight_to_token_ratio=0.041534 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.248004390e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=718127104 dispatch_meta_bytes=458720 combine_payload_bytes=718127104 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=718585824 ingress_bytes=869122048 link_bytes=869122048 token_bytes=1436712928 weight_movement_bytes=150994944 total_nvlink_bytes=1587707872 weight_to_token_ratio=0.105098 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.102169095e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117714 payload_rows_remote=101332 dup_rows=13358 dup_rows_remote=7280 dedup_ratio=0.101913 router_tokens_per_expert=[0, 0, 0, 0, 8190, 0, 0, 2292, 0, 0, 1, 0, 166, 0, 0, 0, 8191, 0, 8180, 10, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7386, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3493, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1679, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1361, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[4646, 16380, 14774, 16382, 16384, 16384, 16382, 16382] dup_rows_per_dst=[272, 6, 0, 12, 6988, 0, 6078, 2] combine_rows_per_src=[4646, 16380, 14774, 16382, 16384, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=904282112 dispatch_meta_bytes=458752 combine_payload_bytes=904282112 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=904740864 ingress_bytes=1281769472 link_bytes=1281769472 token_bytes=1809022976 weight_movement_bytes=377487360 total_nvlink_bytes=2186510336 weight_to_token_ratio=0.208669 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=59520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.625464113e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=933625856 dispatch_meta_bytes=465448 combine_payload_bytes=933625856 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=934091304 ingress_bytes=1009123328 link_bytes=1009123328 token_bytes=1867717160 weight_movement_bytes=75497472 total_nvlink_bytes=1943214632 weight_to_token_ratio=0.040422 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=59520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.279710425e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=836042752 dispatch_meta_bytes=458752 combine_payload_bytes=836042752 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=836501504 ingress_bytes=911540224 link_bytes=911540224 token_bytes=1672544256 weight_movement_bytes=75497472 total_nvlink_bytes=1748041728 weight_to_token_ratio=0.045139 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.155961313e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=802734080 dispatch_meta_bytes=458592 combine_payload_bytes=802734080 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=803192672 ingress_bytes=1029226496 link_bytes=1029226496 token_bytes=1605926752 weight_movement_bytes=226492416 total_nvlink_bytes=1832419168 weight_to_token_ratio=0.141035 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.053309 gemm_active_groups=7 gemm_peak_group_rows=57344 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.305204072e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=830111744 dispatch_meta_bytes=434448 combine_payload_bytes=830111744 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=830546192 ingress_bytes=905609216 link_bytes=905609216 token_bytes=1660657936 weight_movement_bytes=75497472 total_nvlink_bytes=1736155408 weight_to_token_ratio=0.045462 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=33280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.148439960e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116046 payload_rows_remote=99662 dup_rows=15026 dup_rows_remote=5458 dedup_ratio=0.114639 router_tokens_per_expert=[0, 0, 0, 0, 8189, 0, 0, 1134, 0, 0, 2, 0, 299, 0, 0, 0, 8190, 0, 8181, 11, 0, 0, 1, 0, 0, 0, 0, 2, 0, 2, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7565, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4781, 0, 0, 0, 2, 0, 1, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2529, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 58, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[2624, 16382, 15134, 16380, 16384, 16384, 16380, 16378] dup_rows_per_dst=[246, 12, 0, 26, 9568, 0, 5172, 2] combine_rows_per_src=[2624, 16382, 15134, 16380, 16384, 16384, 16380, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118156 payload_rows_remote=102690 dup_rows=12916 dup_rows_remote=12076 dedup_ratio=0.098541 router_tokens_per_expert=[0, 0, 0, 0, 8190, 1, 0, 2387, 0, 0, 1, 0, 151, 0, 0, 0, 8191, 0, 8184, 4, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7388, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3616, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1162, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 15466, 16382, 15320, 16384, 5460, 16382] dup_rows_per_dst=[5076, 2, 840, 20, 6770, 0, 204, 4] combine_rows_per_src=[16384, 16378, 15466, 16382, 15320, 16384, 5460, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21040479512 prefetch_bytes=981467136 reduce_bytes=377487360 weight_movement_bytes=1358954496 egress_bytes=10522967320 ingress_bytes=11876466688 link_bytes=11876466688 total_nvlink_bytes=22399434008 weight_to_token_ratio=0.064588 prefetch_share_of_ingress=0.082640 dedup_ratio=0.065997 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=5 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.052237 gemm_inner_pad_upper_bound=8890 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.255085958e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119198 payload_rows_remote=102814 dup_rows=11874 dup_rows_remote=11870 dedup_ratio=0.090591 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 2257, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4196, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4514, 16384, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 3478, 0, 8392, 4] combine_rows_per_src=[4514, 16384, 16384, 16384, 16380, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120566 payload_rows_remote=104182 dup_rows=10506 dup_rows_remote=10506 dedup_ratio=0.080154 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 2940, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 952, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4294, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5880, 16384, 16384, 16384, 16384, 16384, 16384, 16382] dup_rows_per_dst=[0, 0, 0, 0, 1918, 0, 8588, 0] combine_rows_per_src=[5880, 16384, 16384, 16384, 16384, 16384, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123162 payload_rows_remote=106778 dup_rows=7910 dup_rows_remote=7910 dedup_ratio=0.060349 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 3284, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 481, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4427, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15786, 16382, 16384, 16384, 16384, 16384, 9074, 16384] dup_rows_per_dst=[6374, 0, 0, 0, 964, 0, 572, 0] combine_rows_per_src=[15786, 16382, 16384, 16384, 16384, 16384, 9074, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110090 payload_rows_remote=93706 dup_rows=20982 dup_rows_remote=58 dedup_ratio=0.160080 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 2270, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1371, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4551, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16334, 2742, 16376, 9102, 16384] dup_rows_per_dst=[20924, 8, 50, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16334, 2742, 16376, 9102, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=816431104 dispatch_meta_bytes=420480 combine_payload_bytes=816431104 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=816851584 ingress_bytes=1118420992 link_bytes=1118420992 token_bytes=1633282688 weight_movement_bytes=301989888 total_nvlink_bytes=1935272576 weight_to_token_ratio=0.184898 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=48384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.418315248e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126678 payload_rows_remote=110294 dup_rows=4394 dup_rows_remote=3454 dedup_ratio=0.033524 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 470, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 0, 13, 0, 0, 0, 6323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1728, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7852, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 12672, 16382, 16384, 15704, 16384] dup_rows_per_dst=[0, 940, 0, 2, 3452, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 12672, 16382, 16384, 15704, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=841236480 dispatch_meta_bytes=459064 combine_payload_bytes=841236480 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=841695544 ingress_bytes=992231424 link_bytes=992231424 token_bytes=1682932024 weight_movement_bytes=150994944 total_nvlink_bytes=1833926968 weight_to_token_ratio=0.089721 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=63360 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.258289113e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=842252288 dispatch_meta_bytes=458736 combine_payload_bytes=842252288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=842711024 ingress_bytes=917749760 link_bytes=917749760 token_bytes=1684963312 weight_movement_bytes=75497472 total_nvlink_bytes=1760460784 weight_to_token_ratio=0.044807 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.163835879e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=853458944 dispatch_meta_bytes=458752 combine_payload_bytes=853458944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=853917696 ingress_bytes=928956416 link_bytes=928956416 token_bytes=1707376640 weight_movement_bytes=75497472 total_nvlink_bytes=1782874112 weight_to_token_ratio=0.044218 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178047496e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=874725376 dispatch_meta_bytes=458752 combine_payload_bytes=874725376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=875184128 ingress_bytes=950222848 link_bytes=950222848 token_bytes=1749909504 weight_movement_bytes=75497472 total_nvlink_bytes=1825406976 weight_to_token_ratio=0.043144 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.205016325e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21204723208 prefetch_bytes=830472192 reduce_bytes=1132462080 weight_movement_bytes=1962934272 egress_bytes=10605094408 ingress_bytes=12562563072 link_bytes=12562563072 total_nvlink_bytes=23167657480 weight_to_token_ratio=0.092571 prefetch_share_of_ingress=0.066107 dedup_ratio=0.062414 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=15 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.052543 gemm_inner_pad_upper_bound=9144 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.327591524e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=767639552 dispatch_meta_bytes=375056 combine_payload_bytes=767639552 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=768014608 ingress_bytes=918634496 link_bytes=918634496 token_bytes=1535654160 weight_movement_bytes=150994944 total_nvlink_bytes=1686649104 weight_to_token_ratio=0.098326 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=32384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.164957849e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=903528448 dispatch_meta_bytes=454992 combine_payload_bytes=903528448 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=903983440 ingress_bytes=1205518336 link_bytes=1205518336 token_bytes=1807511888 weight_movement_bytes=301989888 total_nvlink_bytes=2109501776 weight_to_token_ratio=0.167075 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=57344 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.528766940e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21039469792 prefetch_bytes=830472192 reduce_bytes=679477248 weight_movement_bytes=1509949440 egress_bytes=10522465504 ingress_bytes=12026953728 link_bytes=12026953728 total_nvlink_bytes=22549419232 weight_to_token_ratio=0.071767 prefetch_share_of_ingress=0.069051 dedup_ratio=0.065046 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=9 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=9525 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.270989187e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20123632248 prefetch_bytes=754974720 reduce_bytes=1283457024 weight_movement_bytes=2038431744 egress_bytes=10064478840 ingress_bytes=12097585152 link_bytes=12097585152 total_nvlink_bytes=22162063992 weight_to_token_ratio=0.101295 prefetch_share_of_ingress=0.062407 dedup_ratio=0.094969 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=17 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051317 gemm_inner_pad_upper_bound=9525 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.278453403e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21430318240 prefetch_bytes=981467136 reduce_bytes=452984832 weight_movement_bytes=1434451968 egress_bytes=10717901984 ingress_bytes=12146868224 link_bytes=12146868224 total_nvlink_bytes=22864770208 weight_to_token_ratio=0.066936 prefetch_share_of_ingress=0.080800 dedup_ratio=0.044285 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=6 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.052390 gemm_inner_pad_upper_bound=8636 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.283661559e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21744500104 prefetch_bytes=905969664 reduce_bytes=452984832 weight_movement_bytes=1358954496 egress_bytes=10874994056 ingress_bytes=12228460544 link_bytes=12228460544 total_nvlink_bytes=23103454600 weight_to_token_ratio=0.062496 prefetch_share_of_ingress=0.074087 dedup_ratio=0.032677 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=6 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051930 gemm_inner_pad_upper_bound=9398 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.292284105e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19745592968 prefetch_bytes=679477248 reduce_bytes=905969664 weight_movement_bytes=1585446912 egress_bytes=9875445384 ingress_bytes=11455594496 link_bytes=11455594496 total_nvlink_bytes=21331039880 weight_to_token_ratio=0.080294 prefetch_share_of_ingress=0.059314 dedup_ratio=0.109085 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=12 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051777 gemm_inner_pad_upper_bound=9652 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.210608860e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112580 payload_rows_remote=96228 dup_rows=18492 dup_rows_remote=18474 dedup_ratio=0.141083 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 1047, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 703, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6450, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16352, 16384, 16384, 1408, 16384, 12900, 16384] dup_rows_per_dst=[15578, 18, 2888, 0, 0, 8, 0, 0] combine_rows_per_src=[16384, 16352, 16384, 16384, 1408, 16384, 12900, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=788299776 dispatch_meta_bytes=458808 combine_payload_bytes=788299776 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=788758584 ingress_bytes=939294720 link_bytes=939294720 token_bytes=1577058360 weight_movement_bytes=150994944 total_nvlink_bytes=1728053304 weight_to_token_ratio=0.095745 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.191157921e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20717934704 prefetch_bytes=679477248 reduce_bytes=1358954496 weight_movement_bytes=2038431744 egress_bytes=10361706608 ingress_bytes=12394659840 link_bytes=12394659840 total_nvlink_bytes=22756366448 weight_to_token_ratio=0.098390 prefetch_share_of_ingress=0.054820 dedup_ratio=0.073018 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=18 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050245 gemm_inner_pad_upper_bound=11049 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.309847780e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none + [2026-08-29 23:35:13] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 372.2 | throughput per GPU (TFLOP/s/GPU): 485.9 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.544642E+00 | loss scale: 1.0 | grad norm: 5.300 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=94646 payload_rows_remote=78264 dup_rows=36426 dup_rows_remote=22144 dedup_ratio=0.277908 router_tokens_per_expert=[0, 0, 0, 0, 0, 102, 0, 1, 0, 5371, 0, 0, 3, 0, 2, 0, 5, 0, 0, 0, 2861, 0, 0, 1451, 1539, 0, 0, 8191, 52, 1362, 0, 0, 0, 1, 0, 2, 1, 7, 0, 0, 0, 0, 0, 3097, 0, 1, 10, 6625, 2, 1, 1, 2, 0, 0, 0, 0, 0, 1, 2, 2645, 0, 0, 0, 0, 8187, 0, 0, 0, 994, 0, 1, 0, 2718, 0, 2408, 5027, 0, 0, 697, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2333, 0, 1, 0, 0, 642, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 4905, 2, 0, 1, 0, 0, 0, 20, 0, 0, 1, 0, 4253, 0, 0, 0, 1, 5, 0, 0, 0, 0] payload_rows_per_dst=[10824, 7890, 10702, 5298, 13334, 16376, 13840, 16382] dup_rows_per_dst=[134, 928, 2960, 10, 6128, 5952, 6032, 14282] combine_rows_per_src=[10824, 7890, 10702, 5298, 13334, 16376, 13840, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100074 payload_rows_remote=85858 dup_rows=30998 dup_rows_remote=27124 dedup_ratio=0.236496 router_tokens_per_expert=[0, 0, 0, 0, 1, 18, 0, 0, 0, 5854, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2248, 0, 0, 1311, 3234, 0, 0, 8191, 14, 1407, 7, 0, 0, 0, 0, 0, 0, 6, 0, 1, 4, 0, 0, 2119, 0, 0, 12, 6903, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2887, 0, 0, 0, 0, 8192, 0, 0, 0, 1075, 0, 0, 0, 2695, 0, 2270, 5092, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2285, 0, 0, 1, 0, 389, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 4165, 23, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 4487, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[12998, 11686, 14216, 16382, 13376, 8952, 13490, 8974] dup_rows_per_dst=[2242, 4756, 3874, 5774, 7688, 1588, 5076, 0] combine_rows_per_src=[12998, 11686, 14216, 16382, 13376, 8952, 13490, 8974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100756 payload_rows_remote=84378 dup_rows=30316 dup_rows_remote=24994 dedup_ratio=0.231293 router_tokens_per_expert=[0, 1, 0, 0, 3, 21, 0, 0, 1, 6271, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 2328, 0, 4, 1256, 2780, 1, 0, 8191, 8, 1146, 0, 0, 1, 0, 0, 0, 4, 6, 0, 0, 1, 1, 0, 2168, 0, 0, 6, 7203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3031, 0, 2, 0, 0, 8183, 0, 0, 0, 1180, 0, 0, 0, 2001, 0, 1796, 5683, 0, 0, 689, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0, 2171, 0, 1, 0, 0, 490, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 4031, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4738, 0, 0, 0, 0, 0, 0, 3, 0, 0] payload_rows_per_dst=[12560, 11100, 14674, 6810, 9374, 16378, 14300, 15560] dup_rows_per_dst=[34, 3948, 4106, 404, 1958, 5322, 5388, 9156] combine_rows_per_src=[12560, 11100, 14674, 6810, 9374, 16378, 14300, 15560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104424 payload_rows_remote=88040 dup_rows=26648 dup_rows_remote=20544 dedup_ratio=0.203308 router_tokens_per_expert=[0, 0, 0, 0, 0, 43, 0, 2, 3, 5479, 0, 0, 1, 0, 0, 0, 2, 2, 0, 0, 2598, 1, 0, 1305, 2676, 0, 0, 8189, 30, 1492, 2, 0, 0, 0, 0, 1, 0, 13, 0, 2, 0, 0, 1, 2699, 0, 0, 9, 7228, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 3048, 0, 0, 0, 3, 8186, 0, 0, 0, 858, 0, 1, 1, 2315, 0, 2299, 4940, 0, 0, 923, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1712, 1, 0, 0, 0, 330, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3988, 33, 0, 13, 0, 0, 0, 8, 0, 0, 0, 0, 5080, 0, 0, 0, 0, 4, 0, 2, 0, 1] payload_rows_per_dst=[11012, 11646, 15044, 16384, 10426, 16378, 13350, 10184] dup_rows_per_dst=[44, 4570, 4862, 6104, 2368, 4090, 4604, 6] combine_rows_per_src=[11012, 11646, 15044, 16384, 10426, 16378, 13350, 10184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98706 payload_rows_remote=85302 dup_rows=32366 dup_rows_remote=27640 dedup_ratio=0.246933 router_tokens_per_expert=[0, 2, 0, 0, 0, 196, 0, 0, 0, 5840, 0, 1, 0, 0, 0, 0, 0, 2, 1, 3, 1801, 0, 0, 1804, 2351, 0, 0, 8186, 49, 1427, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 2, 0, 0, 3287, 1, 0, 8, 6562, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 3029, 0, 0, 0, 0, 8186, 1, 0, 0, 1064, 0, 0, 0, 2057, 0, 2328, 5001, 2, 2, 519, 0, 1, 3, 0, 2, 0, 0, 0, 0, 0, 1779, 1, 0, 0, 0, 748, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4027, 32, 0, 1, 0, 3, 0, 1, 0, 2, 1, 7, 5205, 0, 0, 0, 2, 2, 0, 0, 0, 0] payload_rows_per_dst=[11830, 10572, 14246, 6062, 9828, 16380, 13404, 16384] dup_rows_per_dst=[248, 3668, 5482, 0, 2118, 5060, 4726, 11064] combine_rows_per_src=[11830, 10572, 14246, 6062, 9828, 16380, 13404, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=104222 payload_rows_remote=93818 dup_rows=26850 dup_rows_remote=24678 dedup_ratio=0.204849 router_tokens_per_expert=[0, 0, 0, 0, 1, 52, 0, 0, 0, 5957, 0, 3, 0, 0, 3, 13, 1, 2, 0, 0, 3009, 0, 0, 1727, 2499, 0, 0, 8187, 43, 1476, 3, 0, 1, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 2299, 0, 0, 32, 6823, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 2727, 0, 0, 0, 1, 8185, 0, 0, 0, 823, 1, 1, 0, 2475, 0, 2335, 4799, 1, 0, 651, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2050, 0, 0, 0, 0, 498, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4064, 63, 1, 11, 0, 0, 0, 3, 0, 0, 0, 2, 4692, 0, 0, 0, 0, 6, 3, 0, 0, 0] payload_rows_per_dst=[12002, 12152, 14248, 16378, 10404, 16376, 13254, 9408] dup_rows_per_dst=[56, 5368, 4078, 5456, 2172, 5092, 4624, 4] combine_rows_per_src=[12002, 12152, 14248, 16378, 10404, 16376, 13254, 9408] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 4, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=93758 payload_rows_remote=77378 dup_rows=37314 dup_rows_remote=24784 dedup_ratio=0.284683 router_tokens_per_expert=[0, 0, 0, 0, 0, 26, 0, 0, 1, 6237, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 2555, 1, 0, 1256, 1345, 0, 0, 8190, 37, 944, 2, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 2911, 0, 0, 2, 6533, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3594, 0, 0, 0, 0, 8189, 0, 2, 0, 1011, 0, 0, 0, 2241, 0, 2129, 5159, 1, 0, 443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3201, 0, 0, 0, 0, 684, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3966, 32, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 4832, 0, 0, 0, 0, 1, 0, 0, 0, 2] payload_rows_per_dst=[16380, 15584, 5828, 16384, 14560, 7362, 7990, 9670] dup_rows_per_dst=[12530, 9764, 4, 7184, 7412, 408, 8, 4] combine_rows_per_src=[16380, 15584, 5828, 16384, 14560, 7362, 7990, 9670] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=641138688 dispatch_meta_bytes=401632 combine_payload_bytes=641138688 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=641540320 ingress_bytes=792133632 link_bytes=792133632 token_bytes=1282679008 weight_movement_bytes=150994944 total_nvlink_bytes=1433673952 weight_to_token_ratio=0.117718 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=38272 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.004536947e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=703348736 dispatch_meta_bytes=451928 combine_payload_bytes=703348736 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=703800664 ingress_bytes=854343680 link_bytes=854343680 token_bytes=1407149400 weight_movement_bytes=150994944 total_nvlink_bytes=1558144344 weight_to_token_ratio=0.107306 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.036765 gemm_active_groups=15 gemm_peak_group_rows=44416 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083428044e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=691224576 dispatch_meta_bytes=437488 combine_payload_bytes=691224576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=691662064 ingress_bytes=766722048 link_bytes=766722048 token_bytes=1382886640 weight_movement_bytes=75497472 total_nvlink_bytes=1458384112 weight_to_token_ratio=0.054594 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.723114812e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=721223680 dispatch_meta_bytes=434336 combine_payload_bytes=721223680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=721658016 ingress_bytes=796721152 link_bytes=796721152 token_bytes=1442881696 weight_movement_bytes=75497472 total_nvlink_bytes=1518379168 weight_to_token_ratio=0.052324 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.034926 gemm_active_groups=15 gemm_peak_group_rows=41600 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.010354568e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=698793984 dispatch_meta_bytes=451768 combine_payload_bytes=698793984 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=699245752 ingress_bytes=774291456 link_bytes=774291456 token_bytes=1398039736 weight_movement_bytes=75497472 total_nvlink_bytes=1473537208 weight_to_token_ratio=0.054002 gemm_rows_defined=67200 gemm_rows_charged=69632 gemm_tail_pad_rows=2432 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.034926 gemm_active_groups=15 gemm_peak_group_rows=32896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.819105560e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=768557056 dispatch_meta_bytes=473984 combine_payload_bytes=768557056 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=769031040 ingress_bytes=1070546944 link_bytes=1070546944 token_bytes=1537588096 weight_movement_bytes=301989888 total_nvlink_bytes=1839577984 weight_to_token_ratio=0.196405 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=19456 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.357604216e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=633880576 dispatch_meta_bytes=408648 combine_payload_bytes=633880576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=634289224 ingress_bytes=709378048 link_bytes=709378048 token_bytes=1268169800 weight_movement_bytes=75497472 total_nvlink_bytes=1343667272 weight_to_token_ratio=0.059533 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=46976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.995912174e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=95366 payload_rows_remote=83774 dup_rows=35706 dup_rows_remote=29858 dedup_ratio=0.272415 router_tokens_per_expert=[0, 0, 0, 4, 0, 18, 0, 0, 0, 5884, 0, 0, 0, 0, 0, 2, 0, 1, 0, 0, 2250, 0, 0, 977, 2398, 0, 0, 8192, 8, 1116, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 2435, 0, 0, 7, 7896, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 1, 3020, 0, 0, 0, 5, 8181, 1, 0, 0, 542, 0, 0, 0, 2829, 0, 2356, 5483, 1, 1, 604, 0, 0, 0, 0, 1, 1, 0, 0, 3, 0, 2162, 0, 0, 0, 0, 512, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3630, 7, 0, 0, 1, 0, 0, 2, 0, 0, 0, 2, 4981, 0, 0, 0, 0, 11, 1, 0, 0, 0] payload_rows_per_dst=[16376, 11592, 13480, 16384, 13314, 5174, 9052, 9994] dup_rows_per_dst=[11802, 5848, 3268, 6058, 6612, 184, 1934, 0] combine_rows_per_src=[16376, 11592, 13480, 16384, 13314, 5174, 9052, 9994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116044 payload_rows_remote=100554 dup_rows=15028 dup_rows_remote=7956 dedup_ratio=0.114655 router_tokens_per_expert=[8191, 1, 0, 0, 0, 0, 8189, 0, 16, 0, 0, 8154, 0, 0, 9, 0, 1, 0, 0, 0, 0, 8191, 0, 0, 0, 1624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 7, 0, 1, 0, 0, 0, 0, 0, 7236, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7524, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16316, 3250, 16380, 16384, 15456, 16384, 16384, 15490] dup_rows_per_dst=[34, 0, 0, 2, 7900, 10, 10, 7072] combine_rows_per_src=[16316, 3250, 16380, 16384, 15456, 16384, 16384, 15490] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123916 payload_rows_remote=107532 dup_rows=7156 dup_rows_remote=7154 dedup_ratio=0.054596 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 8192, 0, 107, 0, 0, 7959, 0, 0, 81, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3425, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 5078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15992, 16384, 16384, 16384, 10158, 16382, 16384, 15848] dup_rows_per_dst=[302, 6850, 2, 0, 0, 0, 2, 0] combine_rows_per_src=[15992, 16384, 16384, 16384, 10158, 16382, 16384, 15848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122826 payload_rows_remote=106444 dup_rows=8246 dup_rows_remote=8246 dedup_ratio=0.062912 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 8191, 0, 33, 0, 0, 7735, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 4955, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4064, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7764, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15472, 9910, 16382, 16384, 16384, 16382, 16384, 15528] dup_rows_per_dst=[76, 0, 6, 0, 8130, 0, 34, 0] combine_rows_per_src=[15472, 9910, 16382, 16384, 16384, 16382, 16384, 15528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121954 payload_rows_remote=105572 dup_rows=9118 dup_rows_remote=9114 dedup_ratio=0.069565 router_tokens_per_expert=[8188, 0, 1, 1, 1, 0, 8188, 0, 48, 0, 0, 8147, 0, 0, 19, 0, 0, 0, 0, 1, 0, 8188, 0, 1, 0, 4484, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 8188, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 4083, 1, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7786, 0, 1, 0, 2, 1, 1, 0, 1, 0] payload_rows_per_dst=[16302, 16380, 16378, 16382, 8170, 16382, 16380, 15580] dup_rows_per_dst=[132, 8968, 2, 4, 2, 2, 2, 6] combine_rows_per_src=[16302, 16380, 16378, 16382, 8170, 16382, 16380, 15580] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117348 payload_rows_remote=100970 dup_rows=13724 dup_rows_remote=13716 dedup_ratio=0.104706 router_tokens_per_expert=[8186, 3, 14, 1, 0, 0, 8175, 0, 122, 0, 0, 7728, 0, 2, 2, 4, 0, 0, 1, 0, 3, 8186, 0, 0, 0, 2037, 0, 1, 0, 0, 0, 0, 0, 0, 59, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 8185, 0, 0, 0, 0, 0, 4, 0, 2, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 6710, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2, 0, 0, 1, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 4, 3, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 7896, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15564, 4082, 16380, 16376, 16378, 16376, 16378, 15814] dup_rows_per_dst=[188, 2, 88, 10, 13420, 8, 8, 0] combine_rows_per_src=[15564, 4082, 16380, 16376, 16378, 16376, 16378, 15814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123772 payload_rows_remote=110222 dup_rows=7300 dup_rows_remote=2940 dedup_ratio=0.055695 router_tokens_per_expert=[8191, 0, 0, 0, 0, 0, 8191, 0, 218, 0, 0, 7921, 0, 0, 8, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 4415, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8192, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 112, 1, 0, 0, 0, 0, 0, 0, 3775, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8106, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16204, 12272, 16384, 16384, 13550, 16382, 16384, 16212] dup_rows_per_dst=[90, 2802, 0, 4, 4360, 0, 44, 0] combine_rows_per_src=[16204, 12272, 16384, 16384, 13550, 16382, 16384, 16212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121734 payload_rows_remote=105450 dup_rows=9338 dup_rows_remote=7544 dedup_ratio=0.071243 router_tokens_per_expert=[8189, 0, 0, 2, 0, 2, 8190, 1, 2, 0, 0, 8127, 0, 0, 24, 0, 0, 0, 0, 0, 0, 8190, 2, 0, 0, 4136, 0, 2, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8188, 8190, 0, 0, 0, 0, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 5531, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6739, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16284, 15578, 16284, 16372, 11066, 16378, 16290, 13482] dup_rows_per_dst=[1794, 7542, 0, 2, 0, 0, 0, 0] combine_rows_per_src=[16284, 15578, 16284, 16372, 11066, 16378, 16290, 13482] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=823738368 dispatch_meta_bytes=434040 combine_payload_bytes=823738368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=824172408 ingress_bytes=899235840 link_bytes=899235840 token_bytes=1647910776 weight_movement_bytes=75497472 total_nvlink_bytes=1723408248 weight_to_token_ratio=0.045814 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.140357622e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=871989248 dispatch_meta_bytes=458760 combine_payload_bytes=871989248 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=872448008 ingress_bytes=1022984192 link_bytes=1022984192 token_bytes=1744437256 weight_movement_bytes=150994944 total_nvlink_bytes=1895432200 weight_to_token_ratio=0.086558 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.297287951e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=880902144 dispatch_meta_bytes=458744 combine_payload_bytes=880902144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=881360888 ingress_bytes=956399616 link_bytes=956399616 token_bytes=1762263032 weight_movement_bytes=75497472 total_nvlink_bytes=1837760504 weight_to_token_ratio=0.042841 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212849337e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=864845824 dispatch_meta_bytes=458744 combine_payload_bytes=864845824 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=865304568 ingress_bytes=1015840768 link_bytes=1015840768 token_bytes=1730150392 weight_movement_bytes=150994944 total_nvlink_bytes=1881145336 weight_to_token_ratio=0.087273 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.288229084e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=686276608 dispatch_meta_bytes=454528 combine_payload_bytes=686276608 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=686731136 ingress_bytes=988266496 link_bytes=988266496 token_bytes=1373007744 weight_movement_bytes=301989888 total_nvlink_bytes=1674997632 weight_to_token_ratio=0.219948 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2032 gemm_tail_pad_fraction=0.036765 gemm_active_groups=16 gemm_peak_group_rows=18944 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.253261026e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=827146240 dispatch_meta_bytes=458744 combine_payload_bytes=827146240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=827604984 ingress_bytes=902643712 link_bytes=902643712 token_bytes=1654751224 weight_movement_bytes=75497472 total_nvlink_bytes=1730248696 weight_to_token_ratio=0.045625 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.144679284e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=902938624 dispatch_meta_bytes=452648 combine_payload_bytes=902938624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=903391272 ingress_bytes=978436096 link_bytes=978436096 token_bytes=1806329896 weight_movement_bytes=75497472 total_nvlink_bytes=1881827368 weight_to_token_ratio=0.041796 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=39552 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.240794695e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=863846400 dispatch_meta_bytes=451976 combine_payload_bytes=863846400 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=864298376 ingress_bytes=1165836288 link_bytes=1165836288 token_bytes=1728144776 weight_movement_bytes=301989888 total_nvlink_bytes=2030134664 weight_to_token_ratio=0.174748 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.038603 gemm_active_groups=14 gemm_peak_group_rows=64000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478444517e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113962 payload_rows_remote=99324 dup_rows=17110 dup_rows_remote=13200 dedup_ratio=0.130539 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1868, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5576, 0, 0, 0, 0, 545, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3698, 0, 0, 0, 0, 0, 0, 0, 0, 4695, 0, 0, 0, 0, 0, 10, 0, 0, 8191, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 1090, 16316, 16384, 14638, 16384, 16382] dup_rows_per_dst=[3736, 54, 0, 0, 0, 3910, 9410, 0] combine_rows_per_src=[16384, 16384, 1090, 16316, 16384, 14638, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101768 payload_rows_remote=85384 dup_rows=29304 dup_rows_remote=12762 dedup_ratio=0.223572 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 1059, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 7524, 0, 0, 0, 0, 964, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4657, 0, 0, 0, 0, 0, 1, 0, 0, 2190, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8191, 0, 1, 0, 1, 0, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[2120, 16252, 1928, 16354, 16356, 15990, 16384, 16384] dup_rows_per_dst=[2, 4, 0, 0, 2, 8374, 4380, 16542] combine_rows_per_src=[2120, 16252, 1928, 16354, 16356, 15990, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117764 payload_rows_remote=101548 dup_rows=13308 dup_rows_remote=4378 dedup_ratio=0.101532 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6014, 0, 0, 0, 0, 839, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5257, 0, 0, 0, 0, 0, 0, 0, 0, 2827, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16216, 16348, 16384, 10514, 9150, 16384] dup_rows_per_dst=[2926, 4, 8930, 0, 0, 0, 1448, 0] combine_rows_per_src=[16384, 16384, 16216, 16348, 16384, 10514, 9150, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118188 payload_rows_remote=101808 dup_rows=12884 dup_rows_remote=8694 dedup_ratio=0.098297 router_tokens_per_expert=[0, 5, 0, 0, 0, 0, 0, 608, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8184, 0, 7611, 0, 0, 0, 0, 3831, 0, 0, 0, 0, 0, 0, 8185, 1, 0, 0, 5, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 6, 8101, 0, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2376, 0, 0, 4, 0, 0, 1, 3, 0, 2091, 0, 0, 1, 0, 0, 0, 0, 0, 8117, 0, 0, 0, 8185, 0, 6, 0, 0, 1, 0, 1, 0, 8186, 0, 0] payload_rows_per_dst=[13008, 16372, 7664, 16222, 16384, 15910, 16380, 16248] dup_rows_per_dst=[964, 0, 0, 12, 18, 4064, 4190, 3636] combine_rows_per_src=[13008, 16372, 7664, 16222, 16384, 15910, 16380, 16248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120240 payload_rows_remote=103860 dup_rows=10832 dup_rows_remote=10830 dedup_ratio=0.082642 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1264, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 6029, 0, 0, 0, 0, 2453, 0, 0, 0, 0, 0, 0, 8189, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 4573, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 0, 0, 0, 2, 0, 0, 8190, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 1] payload_rows_per_dst=[16382, 16382, 11754, 16380, 16384, 10196, 16380, 16382] dup_rows_per_dst=[2528, 6, 3130, 2, 2, 1040, 4122, 2] combine_rows_per_src=[16382, 16382, 11754, 16380, 16384, 10196, 16380, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114898 payload_rows_remote=98514 dup_rows=16174 dup_rows_remote=16174 dedup_ratio=0.123398 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6195, 0, 0, 0, 0, 573, 0, 0, 0, 0, 3, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5104, 0, 0, 0, 0, 0, 0, 0, 0, 3416, 0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1] payload_rows_per_dst=[16384, 16384, 1152, 16356, 16384, 15470, 16384, 16384] dup_rows_per_dst=[2204, 8, 0, 0, 0, 7128, 6832, 2] combine_rows_per_src=[16384, 16384, 1152, 16356, 16384, 15470, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=95342 payload_rows_remote=78958 dup_rows=35730 dup_rows_remote=34830 dedup_ratio=0.272598 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 449, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 6515, 0, 0, 0, 0, 2743, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 5411, 0, 0, 0, 0, 0, 1, 0, 0, 1321, 0, 0, 1, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 0, 8190, 0, 0] payload_rows_per_dst=[16384, 16382, 16382, 16334, 16384, 10826, 2646, 4] dup_rows_per_dst=[900, 0, 34826, 0, 4, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16382, 16334, 16384, 10826, 2646, 4] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120364 payload_rows_remote=103980 dup_rows=10708 dup_rows_remote=40 dedup_ratio=0.081696 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 8147, 0, 0, 6, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 5334, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 3046, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8019, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16306, 16384, 16384, 16384, 6098, 16384, 16384, 16040] dup_rows_per_dst=[16, 10668, 18, 0, 0, 2, 4, 0] combine_rows_per_src=[16306, 16384, 16384, 16384, 6098, 16384, 16384, 16040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=813662208 dispatch_meta_bytes=450096 combine_payload_bytes=813662208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=814112304 ingress_bytes=889159680 link_bytes=889159680 token_bytes=1627774512 weight_movement_bytes=75497472 total_nvlink_bytes=1703271984 weight_to_token_ratio=0.046381 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=37632 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.127579633e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=699465728 dispatch_meta_bytes=392584 combine_payload_bytes=699465728 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=699858312 ingress_bytes=1076953088 link_bytes=1076953088 token_bytes=1399324040 weight_movement_bytes=377487360 total_nvlink_bytes=1776811400 weight_to_token_ratio=0.269764 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=55552 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.365728108e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=831881216 dispatch_meta_bytes=423704 combine_payload_bytes=831881216 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=832304920 ingress_bytes=1058373632 link_bytes=1058373632 token_bytes=1664186136 weight_movement_bytes=226492416 total_nvlink_bytes=1890678552 weight_to_token_ratio=0.136098 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=22144 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.342166743e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=834011136 dispatch_meta_bytes=442008 combine_payload_bytes=834011136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=834453144 ingress_bytes=909508608 link_bytes=909508608 token_bytes=1668464280 weight_movement_bytes=75497472 total_nvlink_bytes=1743961752 weight_to_token_ratio=0.045250 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.153384938e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=850821120 dispatch_meta_bytes=458760 combine_payload_bytes=850821120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=851279880 ingress_bytes=926318592 link_bytes=926318592 token_bytes=1702101000 weight_movement_bytes=75497472 total_nvlink_bytes=1777598472 weight_to_token_ratio=0.044355 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.174702364e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=807026688 dispatch_meta_bytes=458752 combine_payload_bytes=807026688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=807485440 ingress_bytes=882524160 link_bytes=882524160 token_bytes=1614512128 weight_movement_bytes=75497472 total_nvlink_bytes=1690009600 weight_to_token_ratio=0.046762 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.119164860e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=646823936 dispatch_meta_bytes=455152 combine_payload_bytes=646823936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=647279088 ingress_bytes=722321408 link_bytes=722321408 token_bytes=1294103024 weight_movement_bytes=75497472 total_nvlink_bytes=1369600496 weight_to_token_ratio=0.058340 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=55552 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.160052198e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=851804160 dispatch_meta_bytes=416080 combine_payload_bytes=851804160 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=852220240 ingress_bytes=1153794048 link_bytes=1153794048 token_bytes=1704024400 weight_movement_bytes=301989888 total_nvlink_bytes=2006014288 weight_to_token_ratio=0.177222 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.044118 gemm_active_groups=11 gemm_peak_group_rows=35072 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.463173262e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130902 payload_rows_remote=114550 dup_rows=170 dup_rows_remote=170 dedup_ratio=0.001297 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16246, 16384, 16384, 16352, 16384, 16384] dup_rows_per_dst=[0, 16, 6, 0, 140, 0, 8, 0] combine_rows_per_src=[16384, 16384, 16246, 16384, 16384, 16352, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130272 payload_rows_remote=114688 dup_rows=800 dup_rows_remote=800 dedup_ratio=0.006104 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7792, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 387, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15584, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 2, 0, 0, 774, 0, 24, 0] combine_rows_per_src=[16384, 16384, 15584, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118190 payload_rows_remote=101806 dup_rows=12882 dup_rows_remote=1686 dedup_ratio=0.098282 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 1, 0, 0, 0, 0, 0, 7363, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 279, 0, 1, 0, 0, 0, 0, 8189, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 544, 0, 0, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1] payload_rows_per_dst=[16382, 16256, 14728, 16234, 14796, 16382, 7028, 16384] dup_rows_per_dst=[0, 10, 2, 2, 428, 150, 1094, 11196] combine_rows_per_src=[16382, 16256, 14728, 16234, 14796, 16382, 7028, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126070 payload_rows_remote=109708 dup_rows=5002 dup_rows_remote=228 dedup_ratio=0.038162 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 23, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 5752, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 3, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 30, 0, 6, 2, 0, 0, 0, 8179, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2387, 0, 0, 1, 8185, 8145, 0, 0, 0, 0, 0, 0, 8185, 8181, 0, 0, 6] payload_rows_per_dst=[16372, 16384, 11518, 16384, 16374, 16372, 16362, 16304] dup_rows_per_dst=[4, 58, 14, 78, 60, 14, 4774, 0] combine_rows_per_src=[16372, 16384, 11518, 16384, 16374, 16372, 16362, 16304] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130638 payload_rows_remote=114256 dup_rows=434 dup_rows_remote=432 dedup_ratio=0.003311 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 7983, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 8187, 1, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 0, 0, 0, 0, 194, 1, 0, 0, 8191, 8189, 0, 0, 0, 0, 0, 0, 8191, 8191, 0, 0, 1] payload_rows_per_dst=[16384, 16384, 15974, 16382, 16378, 16372, 16384, 16380] dup_rows_per_dst=[6, 8, 0, 2, 22, 0, 394, 2] combine_rows_per_src=[16384, 16384, 15974, 16382, 16378, 16372, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130960 payload_rows_remote=114580 dup_rows=112 dup_rows_remote=20 dedup_ratio=0.000854 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 8144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 1, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16292, 16384, 16380, 16368, 16384, 16384] dup_rows_per_dst=[0, 2, 4, 0, 92, 0, 12, 2] combine_rows_per_src=[16384, 16384, 16292, 16384, 16380, 16368, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[2, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119812 payload_rows_remote=103432 dup_rows=11260 dup_rows_remote=11252 dedup_ratio=0.085907 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 6640, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 34, 0, 2, 0, 0, 0, 0, 8188, 0, 0, 0, 8160, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1531, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 2, 0, 8187, 8190, 0, 0, 2] payload_rows_per_dst=[16380, 16384, 16338, 16384, 16358, 16322, 16382, 5264] dup_rows_per_dst=[8, 16, 8062, 70, 38, 0, 3062, 4] combine_rows_per_src=[16380, 16384, 16338, 16384, 16358, 16322, 16382, 5264] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=938393600 dispatch_meta_bytes=458880 combine_payload_bytes=938393600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938852480 ingress_bytes=1013891072 link_bytes=1013891072 token_bytes=1877246080 weight_movement_bytes=75497472 total_nvlink_bytes=1952743552 weight_to_token_ratio=0.040217 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285756595e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114614 payload_rows_remote=98230 dup_rows=16458 dup_rows_remote=16386 dedup_ratio=0.125565 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 2191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 3957, 0, 0, 0, 0, 238, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6506, 0, 0, 0, 0, 0, 0, 0, 0, 3483, 0, 0, 0, 0, 0, 8, 0, 0, 8167, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16362, 16384, 13012, 6982, 12722] dup_rows_per_dst=[4382, 72, 11616, 386, 2, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16362, 16384, 13012, 6982, 12722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=461952 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939986048 ingress_bytes=1015021568 link_bytes=1015021568 token_bytes=1879510144 weight_movement_bytes=75497472 total_nvlink_bytes=1955007616 weight_to_token_ratio=0.040169 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287190223e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=833994752 dispatch_meta_bytes=413968 combine_payload_bytes=833994752 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=834408720 ingress_bytes=1286979584 link_bytes=1286979584 token_bytes=1668403472 weight_movement_bytes=452984832 total_nvlink_bytes=2121388304 weight_to_token_ratio=0.271508 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.632071269e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=898727936 dispatch_meta_bytes=439744 combine_payload_bytes=898727936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=899167680 ingress_bytes=974225408 link_bytes=974225408 token_bytes=1797895616 weight_movement_bytes=75497472 total_nvlink_bytes=1873393088 weight_to_token_ratio=0.041992 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=60928 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.235454950e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=935985152 dispatch_meta_bytes=458752 combine_payload_bytes=935985152 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=936443904 ingress_bytes=1086980096 link_bytes=1086980096 token_bytes=1872429056 weight_movement_bytes=150994944 total_nvlink_bytes=2023424000 weight_to_token_ratio=0.080641 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378443766e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=938639360 dispatch_meta_bytes=458400 combine_payload_bytes=938639360 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=939097760 ingress_bytes=1089634304 link_bytes=1089634304 token_bytes=1877737120 weight_movement_bytes=150994944 total_nvlink_bytes=2028732064 weight_to_token_ratio=0.080413 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381809675e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=847314944 dispatch_meta_bytes=458736 combine_payload_bytes=847314944 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=847773680 ingress_bytes=998309888 link_bytes=998309888 token_bytes=1695088624 weight_movement_bytes=150994944 total_nvlink_bytes=1846083568 weight_to_token_ratio=0.089078 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265997461e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=804700160 dispatch_meta_bytes=458464 combine_payload_bytes=804700160 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=805158624 ingress_bytes=880197632 link_bytes=880197632 token_bytes=1609858784 weight_movement_bytes=75497472 total_nvlink_bytes=1685356256 weight_to_token_ratio=0.046897 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116214495e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130338 payload_rows_remote=113954 dup_rows=734 dup_rows_remote=734 dedup_ratio=0.005600 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 39, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3576, 0, 0, 4882, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16364, 16336, 16384, 15800, 16302, 16384, 16384, 16384] dup_rows_per_dst=[2, 0, 92, 26, 614, 0, 0, 0] combine_rows_per_src=[16364, 16336, 16384, 15800, 16302, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130408 payload_rows_remote=114024 dup_rows=664 dup_rows_remote=616 dedup_ratio=0.005066 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8159, 0, 0, 0, 21, 0, 0, 0, 0, 8192, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2597, 0, 0, 5856, 7916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16318, 16384, 15832, 16338, 16384, 16384, 16384] dup_rows_per_dst=[12, 0, 48, 36, 568, 0, 0, 0] combine_rows_per_src=[16384, 16318, 16384, 15832, 16338, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125600 payload_rows_remote=109218 dup_rows=5472 dup_rows_remote=5344 dedup_ratio=0.041748 router_tokens_per_expert=[0, 0, 8185, 0, 0, 8177, 0, 0, 6, 0, 0, 0, 0, 8185, 1746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 968, 1, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 1, 0, 0, 0, 3, 0, 0, 16, 0, 0, 0, 6, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0, 0, 7846, 7627, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 4, 0, 0, 60, 0, 7, 0, 0, 0, 0, 6378, 0, 0, 7, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16322, 16372, 15304, 15692, 16376, 16382, 12770] dup_rows_per_dst=[3492, 0, 1824, 4, 8, 2, 128, 14] combine_rows_per_src=[16382, 16322, 16372, 15304, 15692, 16376, 16382, 12770] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115478 payload_rows_remote=99882 dup_rows=15594 dup_rows_remote=8530 dedup_ratio=0.118973 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8189, 0, 0, 1, 0, 0, 0, 0, 8187, 486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 509, 0, 1, 0, 0, 8191, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 7913, 7857, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 4, 0, 1, 0, 0, 0, 0, 7797, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[9170, 16206, 9624, 15738, 16382, 16382, 16380, 15596] dup_rows_per_dst=[964, 0, 312, 0, 7228, 14, 12, 7064] combine_rows_per_src=[9170, 16206, 9624, 15738, 16382, 16382, 16380, 15596] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129384 payload_rows_remote=113946 dup_rows=1688 dup_rows_remote=1684 dedup_ratio=0.012878 router_tokens_per_expert=[0, 0, 8188, 0, 0, 8189, 0, 0, 0, 1, 0, 0, 0, 8188, 173, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8162, 0, 0, 1, 597, 0, 0, 0, 1, 8189, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 96, 0, 1, 8155, 7711, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7862, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0] payload_rows_per_dst=[16382, 16326, 16382, 15438, 16368, 16382, 16380, 15726] dup_rows_per_dst=[342, 2, 1200, 4, 138, 0, 2, 0] combine_rows_per_src=[16382, 16326, 16382, 15438, 16368, 16382, 16380, 15726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130464 payload_rows_remote=114244 dup_rows=608 dup_rows_remote=240 dedup_ratio=0.004639 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 43, 0, 0, 0, 0, 8192, 0, 23, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 53, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2767, 0, 0, 5527, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16358, 16332, 16384, 16020, 16220, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 132, 106, 368, 0, 0, 2] combine_rows_per_src=[16358, 16332, 16384, 16020, 16220, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=933511168 dispatch_meta_bytes=458752 combine_payload_bytes=933511168 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=933969920 ingress_bytes=1084506112 link_bytes=1084506112 token_bytes=1867481088 weight_movement_bytes=150994944 total_nvlink_bytes=2018476032 weight_to_token_ratio=0.080855 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.375306406e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117112 payload_rows_remote=100728 dup_rows=13960 dup_rows_remote=11494 dedup_ratio=0.106506 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8189, 0, 0, 2, 0, 0, 0, 0, 8189, 1140, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 8090, 0, 0, 0, 1945, 0, 1, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 31, 0, 0, 8124, 6240, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 26, 0, 2, 0, 0, 0, 0, 7180, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16188, 16122, 14566, 6858, 16384, 16246, 14364] dup_rows_per_dst=[2466, 682, 3416, 7342, 46, 2, 6, 0] combine_rows_per_src=[16384, 16188, 16122, 14566, 6858, 16384, 16246, 14364] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=934084608 dispatch_meta_bytes=458560 combine_payload_bytes=934084608 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=934543168 ingress_bytes=1160577024 link_bytes=1160577024 token_bytes=1868627776 weight_movement_bytes=226492416 total_nvlink_bytes=2095120192 weight_to_token_ratio=0.121208 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=61312 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.471775030e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130956 payload_rows_remote=114572 dup_rows=116 dup_rows_remote=100 dedup_ratio=0.000885 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16300, 16384, 16380, 16356, 16384, 16384] dup_rows_per_dst=[0, 16, 4, 0, 84, 0, 12, 0] combine_rows_per_src=[16384, 16384, 16300, 16384, 16380, 16356, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 2, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=894713856 dispatch_meta_bytes=458248 combine_payload_bytes=894713856 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=895172104 ingress_bytes=970211328 link_bytes=970211328 token_bytes=1789885960 weight_movement_bytes=75497472 total_nvlink_bytes=1865383432 weight_to_token_ratio=0.042180 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.230364531e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=818233344 dispatch_meta_bytes=433648 combine_payload_bytes=818233344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=818666992 ingress_bytes=893730816 link_bytes=893730816 token_bytes=1636900336 weight_movement_bytes=75497472 total_nvlink_bytes=1712397808 weight_to_token_ratio=0.046122 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=62080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.133376477e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=933445632 dispatch_meta_bytes=462520 combine_payload_bytes=933445632 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=933908152 ingress_bytes=1084440576 link_bytes=1084440576 token_bytes=1867353784 weight_movement_bytes=150994944 total_nvlink_bytes=2018348728 weight_to_token_ratio=0.080860 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=60800 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.375223297e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=935886848 dispatch_meta_bytes=457936 combine_payload_bytes=935886848 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=936344784 ingress_bytes=1162379264 link_bytes=1162379264 token_bytes=1872231632 weight_movement_bytes=226492416 total_nvlink_bytes=2098724048 weight_to_token_ratio=0.120975 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=49920 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.474060524e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=825163776 dispatch_meta_bytes=448888 combine_payload_bytes=825163776 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=825612664 ingress_bytes=1202651136 link_bytes=1202651136 token_bytes=1650776440 weight_movement_bytes=377487360 total_nvlink_bytes=2028263800 weight_to_token_ratio=0.228673 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=61952 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.525130926e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=938573824 dispatch_meta_bytes=458688 combine_payload_bytes=938573824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939032512 ingress_bytes=1014071296 link_bytes=1014071296 token_bytes=1877606336 weight_movement_bytes=75497472 total_nvlink_bytes=1953103808 weight_to_token_ratio=0.040209 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285985145e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129934 payload_rows_remote=113550 dup_rows=1138 dup_rows_remote=1138 dedup_ratio=0.008682 router_tokens_per_expert=[0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7600, 0, 31, 0, 0, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 174, 0, 0, 0, 209, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15262, 16384, 16384, 16384, 16384, 16368] dup_rows_per_dst=[12, 0, 0, 352, 770, 0, 4, 0] combine_rows_per_src=[16384, 16384, 15262, 16384, 16384, 16384, 16384, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114748 payload_rows_remote=98364 dup_rows=16324 dup_rows_remote=594 dedup_ratio=0.124542 router_tokens_per_expert=[0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7822, 0, 43, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 291, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 78, 16384, 16376, 16384, 16374] dup_rows_per_dst=[8, 0, 15730, 0, 584, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 78, 16384, 16376, 16384, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111590 payload_rows_remote=95220 dup_rows=19482 dup_rows_remote=10404 dedup_ratio=0.148636 router_tokens_per_expert=[0, 1, 8155, 15, 0, 0, 0, 6, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 6, 0, 0, 167, 0, 0, 0, 0, 0, 3632, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6417, 0, 0, 0, 0, 0, 0, 2, 0, 8184, 0, 0, 0, 7, 8, 0, 0, 0, 1633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 6, 4554, 8170, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8185, 2, 0, 0, 0, 6, 4, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16352, 16362, 346, 13010, 16384, 16382, 16370, 16384] dup_rows_per_dst=[4, 6, 0, 7092, 3284, 8, 9078, 10] combine_rows_per_src=[16352, 16362, 346, 13010, 16384, 16382, 16370, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=106440 payload_rows_remote=90060 dup_rows=24632 dup_rows_remote=24566 dedup_ratio=0.187927 router_tokens_per_expert=[0, 0, 8190, 0, 0, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 390, 0, 3, 0, 0, 0, 5595, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7726, 0, 0, 0, 0, 0, 0, 1, 0, 8189, 0, 0, 0, 1, 5, 0, 0, 0, 416, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2251, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8189, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16372, 16382, 788, 15472, 8280, 16384, 16382, 16380] dup_rows_per_dst=[2, 0, 0, 11170, 840, 0, 12554, 66] combine_rows_per_src=[16372, 16382, 788, 15472, 8280, 16384, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=930201600 dispatch_meta_bytes=458752 combine_payload_bytes=930201600 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=930660352 ingress_bytes=1081196544 link_bytes=1081196544 token_bytes=1860861952 weight_movement_bytes=150994944 total_nvlink_bytes=2011856896 weight_to_token_ratio=0.081142 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.371109408e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126264 payload_rows_remote=109884 dup_rows=4808 dup_rows_remote=4338 dedup_ratio=0.036682 router_tokens_per_expert=[0, 0, 8191, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 7925, 0, 11, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6, 0, 0, 0, 1, 8187, 0, 0, 0, 0, 231, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16002, 11982, 16380, 16382, 16384, 16368] dup_rows_per_dst=[2, 0, 4280, 56, 470, 0, 0, 0] combine_rows_per_src=[16384, 16382, 16002, 11982, 16380, 16382, 16384, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129464 payload_rows_remote=114158 dup_rows=1608 dup_rows_remote=1608 dedup_ratio=0.012268 router_tokens_per_expert=[0, 0, 8191, 0, 0, 3, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 84, 0, 3, 0, 0, 2, 7651, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7924, 0, 0, 0, 1, 0, 0, 0, 1, 8188, 0, 0, 0, 1, 8, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 557, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8188, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16370, 15880, 15306, 16384, 16380, 16382, 16378] dup_rows_per_dst=[10, 2, 144, 0, 334, 0, 1114, 4] combine_rows_per_src=[16384, 16370, 15880, 15306, 16384, 16380, 16382, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120722 payload_rows_remote=104356 dup_rows=10350 dup_rows_remote=10350 dedup_ratio=0.078964 router_tokens_per_expert=[0, 0, 8180, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 1, 0, 0, 84, 0, 0, 0, 0, 0, 4462, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7105, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 7, 0, 0, 0, 690, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4048, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8190, 1, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16366, 16380, 14344, 16384, 16384, 16384, 8096, 16384] dup_rows_per_dst=[0, 36, 0, 8914, 1394, 0, 0, 6] combine_rows_per_src=[16366, 16380, 14344, 16384, 16384, 16384, 8096, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=805797888 dispatch_meta_bytes=395832 combine_payload_bytes=805797888 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=806193720 ingress_bytes=1032290304 link_bytes=1032290304 token_bytes=1611991608 weight_movement_bytes=226492416 total_nvlink_bytes=1838484024 weight_to_token_ratio=0.140505 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=33664 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.309089411e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=780042240 dispatch_meta_bytes=422496 combine_payload_bytes=780042240 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=780464736 ingress_bytes=1082032128 link_bytes=1082032128 token_bytes=1560506976 weight_movement_bytes=301989888 total_nvlink_bytes=1862496864 weight_to_token_ratio=0.193520 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=50176 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.372169046e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129490 payload_rows_remote=113130 dup_rows=1582 dup_rows_remote=1582 dedup_ratio=0.012070 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 4, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2723, 0, 0, 6248, 7426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16368, 16360, 16384, 14852, 16378, 16380, 16384, 16384] dup_rows_per_dst=[2, 0, 12, 4, 1564, 0, 0, 0] combine_rows_per_src=[16368, 16360, 16384, 14852, 16378, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=737771520 dispatch_meta_bytes=458504 combine_payload_bytes=737771520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=738230024 ingress_bytes=813268992 link_bytes=813268992 token_bytes=1476001544 weight_movement_bytes=75497472 total_nvlink_bytes=1551499016 weight_to_token_ratio=0.051150 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.031339558e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=935182336 dispatch_meta_bytes=463064 combine_payload_bytes=935182336 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=935645400 ingress_bytes=1086177280 link_bytes=1086177280 token_bytes=1870827736 weight_movement_bytes=150994944 total_nvlink_bytes=2021822680 weight_to_token_ratio=0.080710 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=28416 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.377425682e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=900169728 dispatch_meta_bytes=456888 combine_payload_bytes=900169728 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=900626616 ingress_bytes=1126662144 link_bytes=1126662144 token_bytes=1800796344 weight_movement_bytes=226492416 total_nvlink_bytes=2027288760 weight_to_token_ratio=0.125773 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=61568 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.428766188e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=854884352 dispatch_meta_bytes=458824 combine_payload_bytes=854884352 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=855343176 ingress_bytes=930381824 link_bytes=930381824 token_bytes=1710227528 weight_movement_bytes=75497472 total_nvlink_bytes=1785725000 weight_to_token_ratio=0.044145 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.179855114e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=926760960 dispatch_meta_bytes=458848 combine_payload_bytes=926760960 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=927219808 ingress_bytes=1002258432 link_bytes=1002258432 token_bytes=1853980768 weight_movement_bytes=75497472 total_nvlink_bytes=1929478240 weight_to_token_ratio=0.040722 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.271004770e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122136 payload_rows_remote=105804 dup_rows=8936 dup_rows_remote=8936 dedup_ratio=0.068176 router_tokens_per_expert=[0, 0, 0, 0, 115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 842, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7369] payload_rows_per_dst=[9364, 16384, 16384, 16156, 16384, 16332, 16384, 14748] dup_rows_per_dst=[1854, 0, 12, 0, 0, 0, 0, 7070] combine_rows_per_src=[9364, 16384, 16384, 16156, 16384, 16332, 16384, 14748] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128330 payload_rows_remote=111946 dup_rows=2742 dup_rows_remote=2726 dedup_ratio=0.020920 router_tokens_per_expert=[0, 0, 0, 0, 322, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1041, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7859] payload_rows_per_dst=[16384, 16384, 16384, 14366, 16384, 16326, 16384, 15718] dup_rows_per_dst=[2726, 0, 16, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 14366, 16384, 16326, 16384, 15718] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116922 payload_rows_remote=100542 dup_rows=14150 dup_rows_remote=14150 dedup_ratio=0.107956 router_tokens_per_expert=[3, 0, 0, 0, 6820, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 1, 5, 1, 72, 2, 8185, 0, 967, 0, 7, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 0, 0, 0, 7, 6, 7, 0, 0, 63, 0, 2435, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 7, 0, 0, 0, 0, 2, 4, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 7078, 0, 0, 0, 0, 7125] payload_rows_per_dst=[13648, 16384, 16384, 5010, 16364, 16384, 16380, 16368] dup_rows_per_dst=[6, 2072, 2, 28, 0, 0, 0, 12042] combine_rows_per_src=[13648, 16384, 16384, 5010, 16364, 16384, 16380, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=866746368 dispatch_meta_bytes=458960 combine_payload_bytes=866746368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=867205328 ingress_bytes=942243840 link_bytes=942243840 token_bytes=1733951696 weight_movement_bytes=75497472 total_nvlink_bytes=1809449168 weight_to_token_ratio=0.043541 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.194897820e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114742 payload_rows_remote=98360 dup_rows=16330 dup_rows_remote=570 dedup_ratio=0.124588 router_tokens_per_expert=[0, 0, 0, 0, 3346, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 80, 0, 8191, 0, 43, 0, 1, 0, 8191, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0, 13, 0, 6324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8025, 2, 0, 0, 0, 6740] payload_rows_per_dst=[6692, 13464, 16384, 12676, 16382, 16384, 16378, 16382] dup_rows_per_dst=[0, 248, 6, 4, 44, 242, 26, 15760] combine_rows_per_src=[6692, 13464, 16384, 12676, 16382, 16384, 16378, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126892 payload_rows_remote=114226 dup_rows=4180 dup_rows_remote=4180 dedup_ratio=0.031891 router_tokens_per_expert=[0, 0, 0, 0, 1940, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 1, 0, 6, 156, 0, 8190, 0, 0, 0, 1, 0, 8191, 0, 0, 5, 0, 0, 1, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 6332, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 1, 0, 0, 0, 1, 0, 0, 8187, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 8087, 0, 0, 0, 0, 8050] payload_rows_per_dst=[16230, 16384, 16384, 12666, 16374, 16378, 16374, 16102] dup_rows_per_dst=[3832, 326, 18, 0, 0, 4, 0, 0] combine_rows_per_src=[16230, 16384, 16384, 12666, 16374, 16378, 16374, 16102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129024 payload_rows_remote=112640 dup_rows=2048 dup_rows_remote=2048 dedup_ratio=0.015625 router_tokens_per_expert=[0, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 953, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7957] payload_rows_per_dst=[16384, 16384, 16384, 14892, 16384, 16302, 16380, 15914] dup_rows_per_dst=[2034, 0, 14, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 14892, 16384, 16302, 16380, 15914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114482 payload_rows_remote=98142 dup_rows=16590 dup_rows_remote=9164 dedup_ratio=0.126572 router_tokens_per_expert=[1, 0, 0, 0, 4589, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 5, 3, 0, 8190, 0, 52, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 0, 29, 0, 6158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7293, 0, 0, 0, 0, 6441] payload_rows_per_dst=[16340, 16384, 16234, 16384, 16372, 16384, 16384, 0] dup_rows_per_dst=[7426, 274, 6, 8884, 0, 0, 0, 0] combine_rows_per_src=[16340, 16384, 16234, 16384, 16372, 16384, 16384, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=917061632 dispatch_meta_bytes=458688 combine_payload_bytes=917061632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=917520320 ingress_bytes=992559104 link_bytes=992559104 token_bytes=1834581952 weight_movement_bytes=75497472 total_nvlink_bytes=1910079424 weight_to_token_ratio=0.041152 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.258704657e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114642 payload_rows_remote=98258 dup_rows=16430 dup_rows_remote=16430 dedup_ratio=0.125351 router_tokens_per_expert=[0, 0, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7741, 0, 22, 0, 0, 0, 218, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 186, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 14438, 16382, 16382, 1938, 16350] dup_rows_per_dst=[24, 0, 15528, 434, 434, 10, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 14438, 16382, 16382, 1938, 16350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=823640064 dispatch_meta_bytes=458768 combine_payload_bytes=823640064 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=824098832 ingress_bytes=899137536 link_bytes=899137536 token_bytes=1647738896 weight_movement_bytes=75497472 total_nvlink_bytes=1723236368 weight_to_token_ratio=0.045819 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.140232959e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=805765120 dispatch_meta_bytes=395720 combine_payload_bytes=805765120 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=806160840 ingress_bytes=1032257536 link_bytes=1032257536 token_bytes=1611925960 weight_movement_bytes=226492416 total_nvlink_bytes=1838418376 weight_to_token_ratio=0.140510 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=45696 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.309047857e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=935739392 dispatch_meta_bytes=473624 combine_payload_bytes=935739392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=936213016 ingress_bytes=1011236864 link_bytes=1011236864 token_bytes=1871952408 weight_movement_bytes=75497472 total_nvlink_bytes=1947449880 weight_to_token_ratio=0.040331 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=52096 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.282390686e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=922746880 dispatch_meta_bytes=458752 combine_payload_bytes=922746880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=923205632 ingress_bytes=998244352 link_bytes=998244352 token_bytes=1845952512 weight_movement_bytes=75497472 total_nvlink_bytes=1921449984 weight_to_token_ratio=0.040899 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265914352e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=803979264 dispatch_meta_bytes=429224 combine_payload_bytes=803979264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=804408488 ingress_bytes=879476736 link_bytes=879476736 token_bytes=1608387752 weight_movement_bytes=75497472 total_nvlink_bytes=1683885224 weight_to_token_ratio=0.046940 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=44672 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.115300297e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=804929536 dispatch_meta_bytes=458752 combine_payload_bytes=804929536 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=805388288 ingress_bytes=955924480 link_bytes=955924480 token_bytes=1610317824 weight_movement_bytes=150994944 total_nvlink_bytes=1761312768 weight_to_token_ratio=0.093767 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212246797e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130882 payload_rows_remote=114498 dup_rows=190 dup_rows_remote=190 dedup_ratio=0.001450 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8097, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3655, 0, 8, 0, 0, 4624, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16194, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 190, 0, 0, 0, 0] combine_rows_per_src=[16194, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114638 dup_rows=50 dup_rows_remote=50 dedup_ratio=0.000381 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 8163, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2046, 0, 8, 0, 0, 6163, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16334, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 50, 0, 0, 0, 0] combine_rows_per_src=[16334, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130774 payload_rows_remote=114404 dup_rows=298 dup_rows_remote=290 dedup_ratio=0.002274 router_tokens_per_expert=[0, 8185, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 8, 8185, 0, 0, 0, 8092, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 7, 2, 0, 0, 7, 0, 1608, 0, 6577, 0, 0, 0, 7, 0, 0, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4, 8180, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 24, 0, 0, 6, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 99, 0, 8184, 0] payload_rows_per_dst=[16370, 16186, 16356, 16384, 16374, 16362, 16370, 16372] dup_rows_per_dst=[32, 0, 18, 26, 8, 8, 8, 198] combine_rows_per_src=[16370, 16186, 16356, 16384, 16374, 16362, 16370, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=937967616 dispatch_meta_bytes=458752 combine_payload_bytes=937967616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938426368 ingress_bytes=1013465088 link_bytes=1013465088 token_bytes=1876393984 weight_movement_bytes=75497472 total_nvlink_bytes=1951891456 weight_to_token_ratio=0.040235 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.042279 gemm_active_groups=11 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285216388e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130976 payload_rows_remote=114592 dup_rows=96 dup_rows_remote=96 dedup_ratio=0.000732 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 1, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3396, 0, 9, 0, 0, 4832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0] payload_rows_per_dst=[16294, 16384, 16382, 16384, 16384, 16384, 16382, 16382] dup_rows_per_dst=[0, 4, 0, 90, 0, 2, 0, 0] combine_rows_per_src=[16294, 16384, 16382, 16384, 16384, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130898 payload_rows_remote=114516 dup_rows=174 dup_rows_remote=140 dedup_ratio=0.001328 router_tokens_per_expert=[0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 1, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 3876, 0, 4314, 0, 0, 4, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0] payload_rows_per_dst=[16362, 16376, 16384, 16244, 16384, 16384, 16382, 16382] dup_rows_per_dst=[0, 0, 2, 10, 2, 110, 16, 34] combine_rows_per_src=[16362, 16376, 16384, 16244, 16384, 16384, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131022 payload_rows_remote=114638 dup_rows=50 dup_rows_remote=14 dedup_ratio=0.000381 router_tokens_per_expert=[0, 8188, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 4, 8179, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 5146, 0, 3007, 0, 0, 55, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 1, 0, 3, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0] payload_rows_per_dst=[16380, 16366, 16382, 16384, 16384, 16366, 16380, 16380] dup_rows_per_dst=[0, 6, 0, 36, 2, 6, 0, 0] combine_rows_per_src=[16380, 16366, 16382, 16384, 16384, 16366, 16380, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130360 payload_rows_remote=113980 dup_rows=712 dup_rows_remote=314 dedup_ratio=0.005432 router_tokens_per_expert=[0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 1529, 0, 6646, 0, 0, 46, 2, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 10, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 8190, 0] payload_rows_per_dst=[16380, 16380, 16380, 15976, 16378, 16314, 16380, 16172] dup_rows_per_dst=[398, 218, 16, 72, 4, 2, 2, 0] combine_rows_per_src=[16380, 16380, 16380, 15976, 16378, 16314, 16380, 16172] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115570 payload_rows_remote=99186 dup_rows=15502 dup_rows_remote=15502 dedup_ratio=0.118271 router_tokens_per_expert=[0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 824, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 13, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8029, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8167, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7458] payload_rows_per_dst=[16384, 16384, 16384, 16348, 16384, 16336, 16382, 968] dup_rows_per_dst=[1768, 0, 76, 13658, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16348, 16384, 16336, 16382, 968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939114496 dispatch_meta_bytes=458752 combine_payload_bytes=939114496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939573248 ingress_bytes=1014611968 link_bytes=1014611968 token_bytes=1878687744 weight_movement_bytes=75497472 total_nvlink_bytes=1954185216 weight_to_token_ratio=0.040186 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286670793e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=937197568 dispatch_meta_bytes=458776 combine_payload_bytes=937197568 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=937656344 ingress_bytes=1088192512 link_bytes=1088192512 token_bytes=1874853912 weight_movement_bytes=150994944 total_nvlink_bytes=2025848856 weight_to_token_ratio=0.080537 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.379981280e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938737664 dispatch_meta_bytes=458752 combine_payload_bytes=938737664 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=939196416 ingress_bytes=1014235136 link_bytes=1014235136 token_bytes=1877934080 weight_movement_bytes=75497472 total_nvlink_bytes=1953431552 weight_to_token_ratio=0.040202 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286192917e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938115072 dispatch_meta_bytes=458624 combine_payload_bytes=938115072 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938573696 ingress_bytes=1089110016 link_bytes=1089110016 token_bytes=1876688768 weight_movement_bytes=150994944 total_nvlink_bytes=2027683712 weight_to_token_ratio=0.080458 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381144804e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939114496 dispatch_meta_bytes=458608 combine_payload_bytes=939114496 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=939573104 ingress_bytes=1316601856 link_bytes=1316601856 token_bytes=1878687600 weight_movement_bytes=377487360 total_nvlink_bytes=2256174960 weight_to_token_ratio=0.200931 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=23552 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.669636479e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=933724160 dispatch_meta_bytes=457176 combine_payload_bytes=933724160 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=934181336 ingress_bytes=1160216576 link_bytes=1160216576 token_bytes=1867905496 weight_movement_bytes=226492416 total_nvlink_bytes=2094397912 weight_to_token_ratio=0.121255 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.471317931e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=812531712 dispatch_meta_bytes=458752 combine_payload_bytes=812531712 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=812990464 ingress_bytes=1190019072 link_bytes=1190019072 token_bytes=1625522176 weight_movement_bytes=377487360 total_nvlink_bytes=2003009536 weight_to_token_ratio=0.232225 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.509111691e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130514 payload_rows_remote=114372 dup_rows=558 dup_rows_remote=558 dedup_ratio=0.004257 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 8191, 0, 8035, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 253, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16070, 16384, 16382, 16384, 16142, 16384, 16384] dup_rows_per_dst=[0, 0, 48, 0, 2, 0, 2, 506] combine_rows_per_src=[16384, 16070, 16384, 16382, 16384, 16142, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=936935424 dispatch_meta_bytes=459720 combine_payload_bytes=936935424 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937395144 ingress_bytes=1087930368 link_bytes=1087930368 token_bytes=1874330568 weight_movement_bytes=150994944 total_nvlink_bytes=2025325512 weight_to_token_ratio=0.080559 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=52608 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.379648844e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130804 payload_rows_remote=114420 dup_rows=268 dup_rows_remote=256 dedup_ratio=0.002045 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8192, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16228, 16384, 16384, 16384, 16272, 16384, 16384] dup_rows_per_dst=[0, 0, 12, 0, 0, 0, 0, 256] combine_rows_per_src=[16384, 16228, 16384, 16384, 16384, 16272, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117498 payload_rows_remote=101128 dup_rows=13574 dup_rows_remote=3284 dedup_ratio=0.103561 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 2, 0, 8185, 0, 0, 9, 0, 0, 0, 0, 0, 6, 1, 0, 2, 0, 0, 33, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7982, 0, 8185, 0, 0, 0, 0, 0, 0, 1739, 0, 0, 0, 0, 0, 7949, 0, 6, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 63, 1, 3232, 0, 0, 0, 8172, 0, 3, 0, 5156, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 6, 6, 6303, 4, 285, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[16374, 16384, 16366, 16302, 15912, 6600, 16370, 13190] dup_rows_per_dst=[18, 84, 0, 3140, 12, 8, 10290, 22] combine_rows_per_src=[16374, 16384, 16366, 16302, 15912, 6600, 16370, 13190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130662 payload_rows_remote=114278 dup_rows=410 dup_rows_remote=410 dedup_ratio=0.003128 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 8189, 0, 8115, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8064, 0, 0, 0, 8192, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 176, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16230, 16384, 16384, 16384, 16128, 16384, 16384] dup_rows_per_dst=[0, 0, 22, 6, 0, 0, 30, 352] combine_rows_per_src=[16384, 16230, 16384, 16384, 16384, 16128, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120952 payload_rows_remote=106092 dup_rows=10120 dup_rows_remote=5856 dedup_ratio=0.077209 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 8191, 0, 0, 0, 0, 0, 0, 456, 0, 0, 0, 0, 0, 8157, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6634, 0, 0, 0, 8191, 0, 0, 0, 1873, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7270, 1, 199, 0, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16366, 16384, 16266, 11086, 16316, 13290, 16384, 14860] dup_rows_per_dst=[0, 2, 0, 878, 822, 402, 3752, 4264] combine_rows_per_src=[16366, 16384, 16266, 11086, 16316, 13290, 16384, 14860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127794 payload_rows_remote=111474 dup_rows=3278 dup_rows_remote=2428 dedup_ratio=0.025009 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 8190, 1, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 7986, 0, 8187, 0, 0, 0, 0, 0, 4, 593, 0, 0, 0, 0, 2, 8166, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6651, 0, 0, 0, 8191, 0, 0, 0, 1077, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 7901, 0, 379, 0, 0, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[16384, 16376, 16384, 16320, 16334, 13304, 16384, 16308] dup_rows_per_dst=[6, 10, 2, 850, 2, 0, 2154, 254] combine_rows_per_src=[16384, 16376, 16384, 16320, 16334, 13304, 16384, 16308] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121428 payload_rows_remote=105048 dup_rows=9644 dup_rows_remote=9638 dedup_ratio=0.073578 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7959, 0, 8189, 0, 0, 0, 0, 0, 0, 1035, 0, 0, 0, 0, 0, 8064, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 3920, 0, 0, 0, 8186, 0, 0, 0, 4583, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 6879, 2, 306, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16382, 16380, 16226, 16132, 16384, 9170, 14374] dup_rows_per_dst=[6, 1064, 0, 726, 4, 7836, 0, 8] combine_rows_per_src=[16380, 16382, 16380, 16226, 16132, 16384, 9170, 14374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=937328640 dispatch_meta_bytes=458704 combine_payload_bytes=937328640 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937787344 ingress_bytes=1088323584 link_bytes=1088323584 token_bytes=1875115984 weight_movement_bytes=150994944 total_nvlink_bytes=2026110928 weight_to_token_ratio=0.080526 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.380147498e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130998 payload_rows_remote=114614 dup_rows=74 dup_rows_remote=74 dedup_ratio=0.000565 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2535, 0, 22, 0, 0, 5672, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0] payload_rows_per_dst=[16328, 16384, 16384, 16384, 16384, 16366, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 74, 0, 0, 0, 0] combine_rows_per_src=[16328, 16384, 16384, 16384, 16384, 16366, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=828440576 dispatch_meta_bytes=417648 combine_payload_bytes=828440576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=828858224 ingress_bytes=903938048 link_bytes=903938048 token_bytes=1657298800 weight_movement_bytes=75497472 total_nvlink_bytes=1732796272 weight_to_token_ratio=0.045555 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=52864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.146320684e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=936165376 dispatch_meta_bytes=458752 combine_payload_bytes=936165376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=936624128 ingress_bytes=1011662848 link_bytes=1011662848 token_bytes=1872789504 weight_movement_bytes=75497472 total_nvlink_bytes=1948286976 weight_to_token_ratio=0.040313 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.282930894e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=869105664 dispatch_meta_bytes=447792 combine_payload_bytes=869105664 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=869553456 ingress_bytes=944603136 link_bytes=944603136 token_bytes=1738659120 weight_movement_bytes=75497472 total_nvlink_bytes=1814156592 weight_to_token_ratio=0.043423 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=61184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.197889739e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=913195008 dispatch_meta_bytes=455608 combine_payload_bytes=913195008 prefetch_bytes=75497472 reduce_bytes=377487360 egress_bytes=913650616 ingress_bytes=1366179840 link_bytes=1366179840 token_bytes=1826845624 weight_movement_bytes=452984832 total_nvlink_bytes=2279830456 weight_to_token_ratio=0.247960 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=61696 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.732508341e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=860553216 dispatch_meta_bytes=458744 combine_payload_bytes=860553216 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=861011960 ingress_bytes=936050688 link_bytes=936050688 token_bytes=1721565176 weight_movement_bytes=75497472 total_nvlink_bytes=1797062648 weight_to_token_ratio=0.043854 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.187044031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115722 payload_rows_remote=99338 dup_rows=15350 dup_rows_remote=15350 dedup_ratio=0.117111 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 746, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7900, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7738, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 1492, 16384, 16148, 16384, 16384, 16162, 16384] dup_rows_per_dst=[0, 0, 0, 4224, 0, 0, 11126, 0] combine_rows_per_src=[16384, 1492, 16384, 16148, 16384, 16384, 16162, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938917888 dispatch_meta_bytes=458752 combine_payload_bytes=938917888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939376640 ingress_bytes=1014415360 link_bytes=1014415360 token_bytes=1878294528 weight_movement_bytes=75497472 total_nvlink_bytes=1953792000 weight_to_token_ratio=0.040195 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286421466e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=813776896 dispatch_meta_bytes=458752 combine_payload_bytes=813776896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=814235648 ingress_bytes=889274368 link_bytes=889274368 token_bytes=1628012544 weight_movement_bytes=75497472 total_nvlink_bytes=1703510016 weight_to_token_ratio=0.046374 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.127725073e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129222 payload_rows_remote=112838 dup_rows=1850 dup_rows_remote=1850 dedup_ratio=0.014114 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0, 0, 925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7767, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7692, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 15534, 16384, 16384, 15384, 16384] dup_rows_per_dst=[0, 1850, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 15534, 16384, 16384, 15384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125328 payload_rows_remote=108944 dup_rows=5744 dup_rows_remote=532 dedup_ratio=0.043823 router_tokens_per_expert=[0, 0, 0, 0, 1, 8181, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 10, 2, 0, 8184, 0, 8133, 2, 3, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 7, 0, 2, 0, 20, 5587, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 7, 0, 0, 153, 0, 0, 7, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 4, 0, 1, 0, 3, 2332, 0, 318, 0, 0, 0, 8184, 0, 0, 0, 7, 6, 0, 0, 2, 0, 8140, 0, 0] payload_rows_per_dst=[16366, 15946, 16352, 11218, 16384, 16380, 16384, 16298] dup_rows_per_dst=[0, 2, 0, 2, 344, 172, 5212, 12] combine_rows_per_src=[16366, 15946, 16352, 11218, 16384, 16380, 16384, 16298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121164 payload_rows_remote=104780 dup_rows=9908 dup_rows_remote=9908 dedup_ratio=0.075592 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8184, 0, 8186, 0, 0, 1078, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7923, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7396, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 8564, 16384, 15904, 16384, 16368, 14792, 16384] dup_rows_per_dst=[2, 1782, 0, 8124, 0, 0, 0, 0] combine_rows_per_src=[16384, 8564, 16384, 15904, 16384, 16368, 14792, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128124 payload_rows_remote=111758 dup_rows=2948 dup_rows_remote=2140 dedup_ratio=0.022491 router_tokens_per_expert=[0, 0, 0, 0, 8, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 8183, 0, 1, 8020, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7302, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 995, 0, 23, 0, 0, 0, 8190, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8182, 0, 0] payload_rows_per_dst=[16382, 16042, 16382, 14606, 15578, 16384, 16384, 16366] dup_rows_per_dst=[16, 0, 0, 0, 70, 32, 2022, 808] combine_rows_per_src=[16382, 16042, 16382, 14606, 15578, 16384, 16384, 16366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128866 payload_rows_remote=113254 dup_rows=2206 dup_rows_remote=2202 dedup_ratio=0.016830 router_tokens_per_expert=[0, 0, 1, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8183, 0, 8191, 0, 0, 1092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 2, 1, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 5, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7480, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8189, 0, 0] payload_rows_per_dst=[16382, 16384, 16380, 15612, 16384, 16382, 14962, 16380] dup_rows_per_dst=[2, 2184, 0, 4, 14, 0, 0, 2] combine_rows_per_src=[16382, 16384, 16380, 15612, 16384, 16382, 14962, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113778 payload_rows_remote=97400 dup_rows=17294 dup_rows_remote=17286 dedup_ratio=0.131943 router_tokens_per_expert=[0, 0, 0, 0, 2, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8190, 0, 8183, 2, 2, 7975, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8, 6891, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 188, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 70, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1225, 0, 62, 0, 0, 0, 8187, 0, 0, 0, 2, 2, 0, 0, 1, 0, 8159, 0, 0] payload_rows_per_dst=[16378, 16384, 16368, 13798, 16384, 16222, 2578, 15666] dup_rows_per_dst=[8, 16850, 54, 0, 374, 0, 4, 4] combine_rows_per_src=[16378, 16384, 16368, 13798, 16384, 16222, 2578, 15666] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=924368896 dispatch_meta_bytes=458752 combine_payload_bytes=924368896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=924827648 ingress_bytes=999866368 link_bytes=999866368 token_bytes=1849196544 weight_movement_bytes=75497472 total_nvlink_bytes=1924694016 weight_to_token_ratio=0.040827 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.267971296e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121544 payload_rows_remote=105686 dup_rows=9528 dup_rows_remote=9528 dedup_ratio=0.072693 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 8189, 0, 7929, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7853, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 571, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15858, 16384, 16384, 16384, 16382, 7384, 16384] dup_rows_per_dst=[2, 0, 56, 4, 0, 8324, 0, 1142] combine_rows_per_src=[16384, 15858, 16384, 16384, 16384, 16382, 7384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=892469248 dispatch_meta_bytes=437904 combine_payload_bytes=892469248 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=892907152 ingress_bytes=967966720 link_bytes=967966720 token_bytes=1785376400 weight_movement_bytes=75497472 total_nvlink_bytes=1860873872 weight_to_token_ratio=0.042287 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=43008 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.227518053e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=858357760 dispatch_meta_bytes=458752 combine_payload_bytes=858357760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=858816512 ingress_bytes=933855232 link_bytes=933855232 token_bytes=1717174272 weight_movement_bytes=75497472 total_nvlink_bytes=1792671744 weight_to_token_ratio=0.043966 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.184259884e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=915521536 dispatch_meta_bytes=455592 combine_payload_bytes=915521536 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=915977128 ingress_bytes=1217511424 link_bytes=1217511424 token_bytes=1831498664 weight_movement_bytes=301989888 total_nvlink_bytes=2133488552 weight_to_token_ratio=0.164887 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.543975863e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=927776768 dispatch_meta_bytes=461824 combine_payload_bytes=927776768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928238592 ingress_bytes=1003274240 link_bytes=1003274240 token_bytes=1856015360 weight_movement_bytes=75497472 total_nvlink_bytes=1931512832 weight_to_token_ratio=0.040677 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=59136 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.272292958e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 4, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129300 payload_rows_remote=112916 dup_rows=1772 dup_rows_remote=1772 dedup_ratio=0.013519 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7399, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 61, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 729, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 14612, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 122, 186, 0, 1464, 0] combine_rows_per_src=[16384, 14612, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=797900800 dispatch_meta_bytes=458744 combine_payload_bytes=797900800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=798359544 ingress_bytes=873398272 link_bytes=873398272 token_bytes=1596260344 weight_movement_bytes=75497472 total_nvlink_bytes=1671757816 weight_to_token_ratio=0.047296 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.107591949e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=865779712 dispatch_meta_bytes=460856 combine_payload_bytes=865779712 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=866240568 ingress_bytes=1016774656 link_bytes=1016774656 token_bytes=1732020280 weight_movement_bytes=150994944 total_nvlink_bytes=1883015224 weight_to_token_ratio=0.087179 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.289413386e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=925007872 dispatch_meta_bytes=458752 combine_payload_bytes=925007872 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=925466624 ingress_bytes=1302495232 link_bytes=1302495232 token_bytes=1850474496 weight_movement_bytes=377487360 total_nvlink_bytes=2227961856 weight_to_token_ratio=0.203995 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.651747294e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115488 payload_rows_remote=99104 dup_rows=15584 dup_rows_remote=15584 dedup_ratio=0.118896 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7742, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 50, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 400, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16384, 800, 16384] dup_rows_per_dst=[0, 15484, 0, 100, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16384, 800, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114412 payload_rows_remote=98030 dup_rows=16660 dup_rows_remote=1972 dedup_ratio=0.127106 router_tokens_per_expert=[0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8183, 1, 0, 3545, 8183, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8157, 0, 0, 0, 0, 0, 1, 1, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5473, 0, 2, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 6, 8173, 0, 8180, 8184, 0, 0, 2, 2, 1, 0, 0, 0, 87, 0, 0, 0, 7, 7237, 0, 0, 7, 0, 0, 0, 0, 7, 0, 7, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 8, 16364, 16370, 16160, 16364, 16382, 16382] dup_rows_per_dst=[0, 0, 0, 74, 1894, 4, 14688, 0] combine_rows_per_src=[16382, 8, 16364, 16370, 16160, 16364, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128840 payload_rows_remote=112456 dup_rows=2232 dup_rows_remote=2232 dedup_ratio=0.017029 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0, 7078, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1082, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 14158, 16382, 16384, 16384, 16384, 16384, 16380] dup_rows_per_dst=[0, 0, 0, 64, 0, 0, 2168, 0] combine_rows_per_src=[16384, 14158, 16382, 16384, 16384, 16384, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128616 payload_rows_remote=112234 dup_rows=2456 dup_rows_remote=2448 dedup_ratio=0.018738 router_tokens_per_expert=[0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 2, 0, 1102, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8110, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 8191, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7167, 0, 0, 0, 0, 4, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15976, 16376, 16382, 16222, 16382, 14512, 16382] dup_rows_per_dst=[4, 2030, 0, 8, 0, 0, 414, 0] combine_rows_per_src=[16384, 15976, 16376, 16382, 16222, 16382, 14512, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114162 payload_rows_remote=97778 dup_rows=16910 dup_rows_remote=16902 dedup_ratio=0.129013 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 4, 0, 2396, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6730, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8190, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 7253, 0, 0, 1, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 10, 16368, 15876, 16376, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 10, 1878, 498, 14516, 8] combine_rows_per_src=[16382, 10, 16368, 15876, 16376, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125664 payload_rows_remote=109280 dup_rows=5408 dup_rows_remote=5408 dedup_ratio=0.041260 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8190, 0, 0, 2432, 8190, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6421, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8189, 0, 8187, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 2, 7507, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16366, 16380, 12846, 15906, 15018, 16384] dup_rows_per_dst=[0, 5278, 64, 20, 0, 0, 46, 0] combine_rows_per_src=[16384, 16380, 16366, 16380, 12846, 15906, 15018, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=811859968 dispatch_meta_bytes=458752 combine_payload_bytes=811859968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=812318720 ingress_bytes=887357440 link_bytes=887357440 token_bytes=1624178688 weight_movement_bytes=75497472 total_nvlink_bytes=1699676160 weight_to_token_ratio=0.046483 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.125294139e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130244 payload_rows_remote=113860 dup_rows=828 dup_rows_remote=4 dedup_ratio=0.006317 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 8192, 0, 0, 412, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7949, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8031, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 15898, 16384, 16364, 16062, 16384] dup_rows_per_dst=[0, 824, 0, 0, 4, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 15898, 16384, 16364, 16062, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=803061760 dispatch_meta_bytes=400008 combine_payload_bytes=803061760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=803461768 ingress_bytes=878559232 link_bytes=878559232 token_bytes=1606523528 weight_movement_bytes=75497472 total_nvlink_bytes=1682021000 weight_to_token_ratio=0.046994 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=33152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.114136773e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=921239552 dispatch_meta_bytes=458752 combine_payload_bytes=921239552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=921698304 ingress_bytes=996737024 link_bytes=996737024 token_bytes=1842937856 weight_movement_bytes=75497472 total_nvlink_bytes=1918435328 weight_to_token_ratio=0.040966 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=59520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.264002848e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=919420928 dispatch_meta_bytes=458728 combine_payload_bytes=919420928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919879656 ingress_bytes=994918400 link_bytes=994918400 token_bytes=1839300584 weight_movement_bytes=75497472 total_nvlink_bytes=1914798056 weight_to_token_ratio=0.041047 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.261696577e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=800997376 dispatch_meta_bytes=458720 combine_payload_bytes=800997376 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=801456096 ingress_bytes=951992320 link_bytes=951992320 token_bytes=1602453472 weight_movement_bytes=150994944 total_nvlink_bytes=1753448416 weight_to_token_ratio=0.094227 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.207260265e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130698 payload_rows_remote=114314 dup_rows=374 dup_rows_remote=4 dedup_ratio=0.002853 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 10, 8009, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16038, 16384, 16384, 16384, 16384, 16370, 16370] dup_rows_per_dst=[2, 0, 0, 0, 2, 370, 0, 0] combine_rows_per_src=[16384, 16038, 16384, 16384, 16384, 16384, 16370, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=895221760 dispatch_meta_bytes=458752 combine_payload_bytes=895221760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=895680512 ingress_bytes=970719232 link_bytes=970719232 token_bytes=1790902272 weight_movement_bytes=75497472 total_nvlink_bytes=1866399744 weight_to_token_ratio=0.042156 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.231008625e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=932741120 dispatch_meta_bytes=455456 combine_payload_bytes=932741120 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=933196576 ingress_bytes=1234731008 link_bytes=1234731008 token_bytes=1865937696 weight_movement_bytes=301989888 total_nvlink_bytes=2167927584 weight_to_token_ratio=0.161844 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=36864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.565812720e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=936460288 dispatch_meta_bytes=457272 combine_payload_bytes=936460288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=936917560 ingress_bytes=1011957760 link_bytes=1011957760 token_bytes=1873377848 weight_movement_bytes=75497472 total_nvlink_bytes=1948875320 weight_to_token_ratio=0.040300 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283304884e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131014 payload_rows_remote=114630 dup_rows=58 dup_rows_remote=58 dedup_ratio=0.000443 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16330, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[0, 0, 0, 0, 2, 56, 0, 0] combine_rows_per_src=[16384, 16330, 16384, 16384, 16384, 16384, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130912 payload_rows_remote=114528 dup_rows=160 dup_rows_remote=160 dedup_ratio=0.001221 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 8191, 0, 12, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 75, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16252, 16384, 16384, 16384, 16384, 16356, 16384] dup_rows_per_dst=[8, 0, 2, 0, 0, 148, 2, 0] combine_rows_per_src=[16384, 16252, 16384, 16384, 16384, 16384, 16356, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127784 payload_rows_remote=111402 dup_rows=3288 dup_rows_remote=3276 dedup_ratio=0.025085 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 8, 0, 0, 1, 0, 266, 0, 2, 0, 8188, 0, 899, 6766, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7806, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 2, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 1179, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1, 0, 8181, 0, 0, 0, 0, 0, 0, 7636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15002, 15612, 16382, 16384, 16384, 16364, 15272] dup_rows_per_dst=[552, 328, 0, 12, 14, 20, 2362, 0] combine_rows_per_src=[16384, 15002, 15612, 16382, 16384, 16384, 16364, 15272] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111104 payload_rows_remote=98960 dup_rows=19968 dup_rows_remote=12886 dedup_ratio=0.152344 router_tokens_per_expert=[2, 15, 0, 0, 8180, 0, 0, 167, 0, 0, 7, 0, 2550, 0, 0, 0, 8184, 0, 1225, 3895, 0, 0, 1, 0, 0, 0, 0, 6, 0, 7, 0, 0, 8009, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 6969, 0, 0, 0, 0, 0, 1, 0, 0, 109, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 1, 7, 0, 0, 0, 0, 8178, 0, 7, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 1318, 0, 0, 0, 0, 0, 0, 5015, 0, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 6984, 0, 0, 0, 0, 1, 0, 4667, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0] payload_rows_per_dst=[16384, 9580, 13956, 16236, 16374, 16382, 12144, 10048] dup_rows_per_dst=[5458, 688, 0, 6, 14, 2636, 7082, 4084] combine_rows_per_src=[16384, 9580, 13956, 16236, 16374, 16382, 12144, 10048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21197228664 prefetch_bytes=1056964608 reduce_bytes=452984832 weight_movement_bytes=1509949440 egress_bytes=10601351800 ingress_bytes=12105826304 link_bytes=12105826304 total_nvlink_bytes=22707178104 weight_to_token_ratio=0.071233 prefetch_share_of_ingress=0.087310 dedup_ratio=0.053974 prefetch_slots=28 unused_prefetch_slots=356 slot_utilization=0.072917 reduce_slots=6 remote_demand=14 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048866 gemm_inner_pad_upper_bound=11811 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.279324315e-04 prefetch_bytes_per_gemm_flop=1.116983252e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 2, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109058 payload_rows_remote=92828 dup_rows=22014 dup_rows_remote=10372 dedup_ratio=0.167953 router_tokens_per_expert=[0, 0, 0, 0, 8191, 0, 0, 119, 0, 0, 1, 0, 650, 0, 0, 0, 8190, 0, 830, 4037, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 7936, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8191, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 0, 0, 0, 5104, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 7560, 0, 0, 0, 0, 0, 0, 6376, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[1496, 16298, 15708, 16382, 16306, 16384, 10254, 16230] dup_rows_per_dst=[832, 9034, 0, 170, 2, 296, 38, 11642] combine_rows_per_src=[1496, 16298, 15708, 16382, 16306, 16384, 10254, 16230] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111282 payload_rows_remote=94898 dup_rows=19790 dup_rows_remote=11758 dedup_ratio=0.150986 router_tokens_per_expert=[0, 0, 0, 0, 8189, 0, 0, 138, 0, 0, 2, 0, 1309, 0, 0, 0, 8190, 0, 1388, 4154, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 8158, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 7671, 0, 0, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 259, 0, 0, 0, 0, 0, 0, 6160, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 7558, 0, 0, 0, 0, 0, 0, 4134, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0] payload_rows_per_dst=[16384, 12236, 15346, 16348, 16384, 12380, 13934, 8270] dup_rows_per_dst=[8032, 2996, 4912, 0, 10, 382, 3454, 4] combine_rows_per_src=[16384, 12236, 15346, 16348, 16384, 12380, 13934, 8270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939048960 dispatch_meta_bytes=458752 combine_payload_bytes=939048960 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939507712 ingress_bytes=1165541376 link_bytes=1165541376 token_bytes=1878556672 weight_movement_bytes=226492416 total_nvlink_bytes=2105049088 weight_to_token_ratio=0.120567 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=63104 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478070527e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116574 payload_rows_remote=100190 dup_rows=14498 dup_rows_remote=130 dedup_ratio=0.110611 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 7184, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 65, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 941, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16358, 16384, 1912, 16384] dup_rows_per_dst=[0, 14368, 0, 130, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16358, 16384, 1912, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938213376 dispatch_meta_bytes=458752 combine_payload_bytes=938213376 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=938672128 ingress_bytes=1013710848 link_bytes=1013710848 token_bytes=1876885504 weight_movement_bytes=75497472 total_nvlink_bytes=1952382976 weight_to_token_ratio=0.040225 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285528046e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=912605184 dispatch_meta_bytes=458712 combine_payload_bytes=912605184 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=913063896 ingress_bytes=1063600128 link_bytes=1063600128 token_bytes=1825669080 weight_movement_bytes=150994944 total_nvlink_bytes=1976664024 weight_to_token_ratio=0.082707 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.348794675e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=810680320 dispatch_meta_bytes=447384 combine_payload_bytes=810680320 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=811127704 ingress_bytes=1112670208 link_bytes=1112670208 token_bytes=1621808024 weight_movement_bytes=301989888 total_nvlink_bytes=1923797912 weight_to_token_ratio=0.186206 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=48128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.411022444e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=760446976 dispatch_meta_bytes=412800 combine_payload_bytes=760446976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=760859776 ingress_bytes=835944448 link_bytes=835944448 token_bytes=1521306752 weight_movement_bytes=75497472 total_nvlink_bytes=1596804224 weight_to_token_ratio=0.049627 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=55680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.060095228e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21134045680 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=10569724400 ingress_bytes=12225265664 link_bytes=12225265664 total_nvlink_bytes=22794990064 weight_to_token_ratio=0.078591 prefetch_share_of_ingress=0.067931 dedup_ratio=0.057095 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049939 gemm_inner_pad_upper_bound=10922 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.291946475e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=777404416 dispatch_meta_bytes=426624 combine_payload_bytes=777404416 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=777831040 ingress_bytes=928399360 link_bytes=928399360 token_bytes=1555235456 weight_movement_bytes=150994944 total_nvlink_bytes=1706230400 weight_to_token_ratio=0.097088 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=57728 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177341071e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=820756480 dispatch_meta_bytes=401280 combine_payload_bytes=820756480 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=821157760 ingress_bytes=1122746368 link_bytes=1122746368 token_bytes=1641914240 weight_movement_bytes=301989888 total_nvlink_bytes=1943904128 weight_to_token_ratio=0.183925 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=32896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.423800433e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21542868944 prefetch_bytes=679477248 reduce_bytes=830472192 weight_movement_bytes=1509949440 egress_bytes=10774190032 ingress_bytes=12278628352 link_bytes=12278628352 total_nvlink_bytes=23052818384 weight_to_token_ratio=0.070090 prefetch_share_of_ingress=0.055338 dedup_ratio=0.044954 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=11 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049326 gemm_inner_pad_upper_bound=11557 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.297585758e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21584214320 prefetch_bytes=981467136 reduce_bytes=981467136 weight_movement_bytes=1962934272 egress_bytes=10794858800 ingress_bytes=12752289792 link_bytes=12752289792 total_nvlink_bytes=23547148592 weight_to_token_ratio=0.090943 prefetch_share_of_ingress=0.076964 dedup_ratio=0.041826 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=13 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048407 gemm_inner_pad_upper_bound=12446 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.347641540e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20063143352 prefetch_bytes=981467136 reduce_bytes=452984832 weight_movement_bytes=1434451968 egress_bytes=10034218424 ingress_bytes=11463376896 link_bytes=11463376896 total_nvlink_bytes=21497595320 weight_to_token_ratio=0.071497 prefetch_share_of_ingress=0.085618 dedup_ratio=0.101295 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=6 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048100 gemm_inner_pad_upper_bound=12065 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.211431291e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19293751912 prefetch_bytes=981467136 reduce_bytes=1207959552 weight_movement_bytes=2189426688 egress_bytes=9649457768 ingress_bytes=11833720832 link_bytes=11833720832 total_nvlink_bytes=21483178600 weight_to_token_ratio=0.113479 prefetch_share_of_ingress=0.082938 dedup_ratio=0.128658 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=16 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048100 gemm_inner_pad_upper_bound=12446 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.250568645e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19486766688 prefetch_bytes=754974720 reduce_bytes=981467136 weight_movement_bytes=1736441856 egress_bytes=9746069088 ingress_bytes=11477139456 link_bytes=11477139456 total_nvlink_bytes=21223208544 weight_to_token_ratio=0.089109 prefetch_share_of_ingress=0.065781 dedup_ratio=0.119139 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=13 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.048560 gemm_inner_pad_upper_bound=12192 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.212885697e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130926 payload_rows_remote=114636 dup_rows=146 dup_rows_remote=146 dedup_ratio=0.001114 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8, 8137, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16290, 16384, 16384, 16384, 16384, 16332, 16384] dup_rows_per_dst=[4, 0, 0, 0, 44, 98, 0, 0] combine_rows_per_src=[16384, 16290, 16384, 16384, 16384, 16384, 16332, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939098112 dispatch_meta_bytes=459128 combine_payload_bytes=939098112 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939557240 ingress_bytes=1090093056 link_bytes=1090093056 token_bytes=1878655352 weight_movement_bytes=150994944 total_nvlink_bytes=2029650296 weight_to_token_ratio=0.080374 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=51328 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382391437e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20651140128 prefetch_bytes=905969664 reduce_bytes=1434451968 weight_movement_bytes=2340421632 egress_bytes=10328269856 ingress_bytes=12663291904 link_bytes=12663291904 total_nvlink_bytes=22991561760 weight_to_token_ratio=0.113331 prefetch_share_of_ingress=0.071543 dedup_ratio=0.077296 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=19 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.047181 gemm_inner_pad_upper_bound=13589 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.338236386e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none + [2026-08-29 23:35:13] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 370.7 | throughput per GPU (TFLOP/s/GPU): 487.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.445278E+00 | loss scale: 1.0 | grad norm: 5.036 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=94804 payload_rows_remote=78420 dup_rows=36268 dup_rows_remote=25716 dedup_ratio=0.276703 router_tokens_per_expert=[0, 0, 0, 0, 0, 92, 0, 0, 0, 5892, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1928, 0, 0, 1129, 2127, 0, 0, 8192, 37, 1155, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1706, 0, 0, 6, 6699, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3380, 0, 0, 0, 0, 8192, 0, 0, 0, 1319, 0, 0, 0, 2261, 0, 2797, 5529, 0, 0, 1987, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1613, 0, 0, 0, 0, 980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3487, 340, 0, 4, 0, 0, 0, 4, 0, 0, 0, 0, 4622, 0, 0, 0, 0, 56, 0, 0, 0, 0] payload_rows_per_dst=[11868, 9782, 8828, 12116, 14498, 4944, 16384, 16384] dup_rows_per_dst=[100, 2972, 1260, 4340, 9138, 244, 7662, 10552] combine_rows_per_src=[11868, 9782, 8828, 12116, 14498, 4944, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=103878 payload_rows_remote=91698 dup_rows=27194 dup_rows_remote=23294 dedup_ratio=0.207474 router_tokens_per_expert=[0, 0, 0, 0, 0, 61, 0, 0, 0, 5465, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1941, 0, 0, 1364, 2824, 0, 0, 8192, 27, 1065, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1835, 0, 0, 8, 6443, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3436, 0, 0, 0, 0, 8190, 0, 0, 0, 1122, 0, 0, 0, 2534, 0, 2747, 5594, 1, 0, 1636, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1685, 1, 0, 0, 0, 1003, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2819, 149, 0, 4, 0, 0, 0, 1, 0, 0, 0, 1, 5354, 0, 0, 0, 0, 30, 0, 0, 0, 0] payload_rows_per_dst=[10970, 10696, 13466, 13060, 12180, 16380, 16384, 10742] dup_rows_per_dst=[82, 3748, 3110, 5000, 3900, 5380, 5944, 30] combine_rows_per_src=[10970, 10696, 13466, 13060, 12180, 16380, 16384, 10742] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=98546 payload_rows_remote=82162 dup_rows=32526 dup_rows_remote=23128 dedup_ratio=0.248154 router_tokens_per_expert=[0, 0, 0, 0, 1, 155, 0, 0, 0, 4543, 0, 0, 0, 0, 0, 0, 0, 3, 0, 2, 1913, 0, 0, 1574, 1798, 0, 0, 8192, 41, 1639, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2522, 0, 0, 28, 6462, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3580, 0, 0, 0, 0, 8190, 0, 0, 0, 1081, 0, 3, 0, 2374, 0, 2619, 4318, 0, 0, 1374, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1888, 0, 0, 2, 0, 1211, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3819, 96, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6061, 0, 0, 0, 0, 26, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11736, 10568, 11902, 11600, 16380, 7824, 12152] dup_rows_per_dst=[9398, 6756, 2928, 3894, 3302, 6206, 20, 22] combine_rows_per_src=[16384, 11736, 10568, 11902, 11600, 16380, 7824, 12152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97774 payload_rows_remote=86620 dup_rows=33298 dup_rows_remote=30414 dedup_ratio=0.254044 router_tokens_per_expert=[2, 1, 0, 0, 1, 60, 1, 0, 0, 5628, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 2344, 0, 1, 1761, 2421, 0, 0, 8183, 34, 2036, 1, 0, 0, 2, 1, 0, 0, 5, 0, 2, 2, 0, 0, 2975, 1, 1, 1, 6199, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 2243, 0, 0, 0, 0, 8184, 0, 0, 0, 966, 1, 1, 0, 2495, 0, 1996, 4774, 0, 0, 1164, 2, 0, 0, 1, 0, 1, 0, 0, 1, 1, 2195, 0, 0, 0, 1, 1048, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 4315, 55, 0, 0, 0, 0, 0, 10, 0, 3, 0, 0, 4399, 0, 0, 0, 1, 5, 0, 0, 0, 0] payload_rows_per_dst=[11294, 14022, 13546, 11154, 10766, 16378, 11804, 8810] dup_rows_per_dst=[92, 12616, 4832, 2884, 2484, 6486, 3878, 26] combine_rows_per_src=[11294, 14022, 13546, 11154, 10766, 16378, 11804, 8810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97060 payload_rows_remote=88638 dup_rows=34012 dup_rows_remote=32146 dedup_ratio=0.259491 router_tokens_per_expert=[0, 0, 0, 1, 0, 333, 0, 1, 1, 3817, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1521, 0, 1, 1253, 3056, 0, 0, 8191, 40, 1622, 0, 0, 0, 0, 0, 0, 1, 1, 0, 2, 0, 0, 0, 2922, 0, 0, 1, 6061, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3124, 0, 0, 1, 0, 8190, 0, 0, 0, 1849, 0, 0, 0, 2144, 0, 2296, 5547, 0, 0, 2110, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 2040, 0, 0, 0, 0, 1049, 2, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 3204, 147, 0, 6, 0, 0, 0, 2, 0, 0, 0, 0, 4988, 0, 0, 1, 0, 5, 0, 0, 0, 0] payload_rows_per_dst=[8100, 11148, 13330, 12924, 12146, 8422, 16384, 14606] dup_rows_per_dst=[208, 3838, 4646, 4422, 4654, 1866, 6716, 7662] combine_rows_per_src=[8100, 11148, 13330, 12924, 12146, 8422, 16384, 14606] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=96192 payload_rows_remote=79810 dup_rows=34880 dup_rows_remote=26658 dedup_ratio=0.266113 router_tokens_per_expert=[0, 0, 0, 6, 0, 242, 0, 0, 1, 5379, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1924, 0, 0, 1479, 1608, 0, 0, 8191, 33, 1937, 1, 1, 0, 1, 0, 0, 0, 2, 0, 2, 2, 0, 0, 3350, 0, 0, 1, 6292, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2816, 0, 0, 0, 0, 8186, 0, 0, 0, 1494, 0, 0, 0, 2384, 0, 1980, 5778, 0, 1, 1399, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1073, 0, 1, 0, 0, 957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3872, 239, 0, 0, 0, 0, 0, 52, 0, 1, 0, 7, 4825, 0, 0, 0, 0, 16, 0, 0, 0, 0] payload_rows_per_dst=[10972, 10510, 13738, 13040, 11198, 3976, 16382, 16376] dup_rows_per_dst=[286, 3458, 5562, 4148, 3318, 88, 8222, 9798] combine_rows_per_src=[10972, 10510, 13738, 13040, 11198, 3976, 16382, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=642416640 dispatch_meta_bytes=416544 combine_payload_bytes=642416640 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=642833184 ingress_bytes=793411584 link_bytes=793411584 token_bytes=1285249824 weight_movement_bytes=150994944 total_nvlink_bytes=1436244768 weight_to_token_ratio=0.117483 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.049632 gemm_active_groups=10 gemm_peak_group_rows=42112 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.006157570e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=751190016 dispatch_meta_bytes=459968 combine_payload_bytes=751190016 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=751649984 ingress_bytes=1128677376 link_bytes=1128677376 token_bytes=1502840000 weight_movement_bytes=377487360 total_nvlink_bytes=1880327360 weight_to_token_ratio=0.251183 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=18432 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.431321786e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=673071104 dispatch_meta_bytes=421160 combine_payload_bytes=673071104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=673492264 ingress_bytes=748568576 link_bytes=748568576 token_bytes=1346563368 weight_movement_bytes=75497472 total_nvlink_bytes=1422060840 weight_to_token_ratio=0.056067 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.036765 gemm_active_groups=14 gemm_peak_group_rows=39680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.492903234e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=100534 payload_rows_remote=86106 dup_rows=30538 dup_rows_remote=26272 dedup_ratio=0.232986 router_tokens_per_expert=[0, 0, 0, 0, 0, 280, 0, 0, 0, 4192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2237, 0, 0, 1618, 1917, 0, 0, 8192, 53, 1468, 1, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 0, 2, 2390, 0, 0, 23, 6920, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3339, 0, 0, 0, 0, 8192, 0, 0, 0, 1521, 0, 0, 0, 2257, 0, 1829, 4712, 0, 0, 1900, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1500, 0, 0, 0, 0, 1496, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3128, 244, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 6012, 0, 0, 0, 0, 96, 0, 0, 0, 0] payload_rows_per_dst=[16314, 11150, 14428, 11850, 11506, 16384, 6718, 12184] dup_rows_per_dst=[8850, 3602, 4266, 4252, 3508, 5996, 32, 32] combine_rows_per_src=[16314, 11150, 14428, 11850, 11506, 16384, 6718, 12184] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=709591040 dispatch_meta_bytes=468136 combine_payload_bytes=709591040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=710059176 ingress_bytes=785088512 link_bytes=785088512 token_bytes=1419650216 weight_movement_bytes=75497472 total_nvlink_bytes=1495147688 weight_to_token_ratio=0.053180 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=40576 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.956027428e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=726122496 dispatch_meta_bytes=483136 combine_payload_bytes=726122496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=726605632 ingress_bytes=801619968 link_bytes=801619968 token_bytes=1452728128 weight_movement_bytes=75497472 total_nvlink_bytes=1528225600 weight_to_token_ratio=0.051969 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.042279 gemm_active_groups=13 gemm_peak_group_rows=43008 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.016566956e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=653803520 dispatch_meta_bytes=425872 combine_payload_bytes=653803520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=654229392 ingress_bytes=729300992 link_bytes=729300992 token_bytes=1308032912 weight_movement_bytes=75497472 total_nvlink_bytes=1383530384 weight_to_token_ratio=0.057718 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=36352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.248563147e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99674 payload_rows_remote=88454 dup_rows=31398 dup_rows_remote=27580 dedup_ratio=0.239548 router_tokens_per_expert=[0, 0, 0, 1, 3, 195, 0, 0, 0, 4685, 0, 0, 1, 0, 0, 3, 2, 2, 0, 4, 1900, 0, 1, 1280, 2368, 0, 0, 8189, 56, 1905, 1, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 2568, 0, 0, 7, 5705, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 3151, 0, 3, 0, 0, 8189, 0, 0, 0, 1696, 0, 1, 0, 2183, 0, 2155, 4887, 1, 0, 2401, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1693, 0, 1, 0, 0, 1087, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3195, 152, 0, 11, 0, 0, 0, 7, 0, 2, 0, 0, 5838, 0, 0, 0, 0, 1, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11220, 12844, 12090, 12356, 16380, 6708, 11692] dup_rows_per_dst=[9770, 3818, 3720, 3996, 4518, 5562, 10, 4] combine_rows_per_src=[16384, 11220, 12844, 12090, 12356, 16380, 6708, 11692] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115474 payload_rows_remote=99136 dup_rows=15598 dup_rows_remote=14002 dedup_ratio=0.119003 router_tokens_per_expert=[8188, 0, 0, 0, 0, 0, 8192, 0, 457, 0, 0, 344, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1313, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 4023, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2094, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[8054, 9170, 16376, 16384, 16384, 16384, 16384, 16338] dup_rows_per_dst=[1432, 270, 56, 0, 8050, 4, 4190, 1596] combine_rows_per_src=[8054, 9170, 16376, 16384, 16384, 16384, 16384, 16338] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125326 payload_rows_remote=114408 dup_rows=5746 dup_rows_remote=5746 dedup_ratio=0.043839 router_tokens_per_expert=[8192, 0, 6, 0, 0, 0, 8187, 0, 66, 0, 0, 391, 0, 0, 8136, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 932, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5459, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 1486, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 16384, 16384, 16380, 10918, 16384, 16384, 16166] dup_rows_per_dst=[872, 1868, 30, 0, 0, 4, 2972, 0] combine_rows_per_src=[16326, 16384, 16384, 16380, 10918, 16384, 16384, 16166] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=705380352 dispatch_meta_bytes=449512 combine_payload_bytes=705380352 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=705829864 ingress_bytes=856375296 link_bytes=856375296 token_bytes=1411210216 weight_movement_bytes=150994944 total_nvlink_bytes=1562205160 weight_to_token_ratio=0.106997 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1651 gemm_tail_pad_fraction=0.040441 gemm_active_groups=13 gemm_peak_group_rows=45184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.086004419e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112546 payload_rows_remote=96240 dup_rows=18526 dup_rows_remote=16628 dedup_ratio=0.141342 router_tokens_per_expert=[8132, 0, 0, 0, 0, 0, 8192, 0, 513, 0, 0, 457, 0, 0, 8132, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 3677, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 3542, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16306, 16384, 16264, 16384, 7356, 16384, 7084, 16384] dup_rows_per_dst=[1898, 240, 0, 16384, 0, 4, 0, 0] combine_rows_per_src=[16306, 16384, 16264, 16384, 7356, 16384, 7084, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122694 payload_rows_remote=106310 dup_rows=8378 dup_rows_remote=8368 dedup_ratio=0.063919 router_tokens_per_expert=[8188, 0, 2, 1, 0, 0, 8183, 0, 189, 0, 0, 938, 0, 0, 8111, 0, 0, 0, 2, 0, 3, 8188, 0, 0, 0, 1657, 0, 2, 0, 0, 1, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8190, 0, 0, 0, 0, 0, 2, 0, 1, 3, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 4374, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 1, 0, 1413, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7852, 7, 2, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[16316, 16384, 16376, 16384, 8752, 16380, 16380, 15722] dup_rows_per_dst=[2166, 3322, 40, 10, 2, 4, 2830, 4] combine_rows_per_src=[16316, 16384, 16376, 16384, 8752, 16380, 16380, 15722] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118884 payload_rows_remote=102504 dup_rows=12188 dup_rows_remote=12188 dedup_ratio=0.092987 router_tokens_per_expert=[8190, 0, 0, 1, 0, 0, 8189, 1, 68, 0, 0, 274, 0, 0, 7953, 0, 0, 0, 0, 1, 0, 8190, 1, 0, 0, 99, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 7527, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 540, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8108, 0, 1, 0, 0, 0, 0, 0, 1, 0] payload_rows_per_dst=[16092, 16384, 16382, 5096, 15946, 16380, 16384, 16220] dup_rows_per_dst=[502, 198, 2, 2, 10400, 0, 1082, 2] combine_rows_per_src=[16092, 16384, 16382, 5096, 15946, 16380, 16384, 16220] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=812122112 dispatch_meta_bytes=452552 combine_payload_bytes=812122112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=812574664 ingress_bytes=887619584 link_bytes=887619584 token_bytes=1624696776 weight_movement_bytes=75497472 total_nvlink_bytes=1700194248 weight_to_token_ratio=0.046469 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.125626574e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 2, 0, 0, 2, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=937230336 dispatch_meta_bytes=480616 combine_payload_bytes=937230336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937710952 ingress_bytes=1012727808 link_bytes=1012727808 token_bytes=1874941288 weight_movement_bytes=75497472 total_nvlink_bytes=1950438760 weight_to_token_ratio=0.040267 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284281413e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=724615168 dispatch_meta_bytes=464136 combine_payload_bytes=724615168 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=725079304 ingress_bytes=951107584 link_bytes=951107584 token_bytes=1449694472 weight_movement_bytes=226492416 total_nvlink_bytes=1676186888 weight_to_token_ratio=0.156235 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.042279 gemm_active_groups=14 gemm_peak_group_rows=18176 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.206138295e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113962 payload_rows_remote=97578 dup_rows=17110 dup_rows_remote=15364 dedup_ratio=0.130539 router_tokens_per_expert=[8192, 0, 20, 0, 0, 0, 8189, 0, 102, 0, 0, 983, 0, 0, 7654, 0, 0, 0, 0, 0, 2, 8192, 0, 0, 0, 167, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6723, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 1, 0, 0, 0, 0, 873, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8035, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15966, 16116, 16384, 268, 16380, 16384, 16384, 16080] dup_rows_per_dst=[1552, 338, 26, 0, 13446, 2, 1746, 0] combine_rows_per_src=[15966, 16116, 16384, 268, 16380, 16384, 16384, 16080] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=788398080 dispatch_meta_bytes=451472 combine_payload_bytes=788398080 prefetch_bytes=0 reduce_bytes=452984832 egress_bytes=788849552 ingress_bytes=1241382912 link_bytes=1241382912 token_bytes=1577247632 weight_movement_bytes=452984832 total_nvlink_bytes=2030232464 weight_to_token_ratio=0.287200 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=59776 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.574248270e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=870891520 dispatch_meta_bytes=458712 combine_payload_bytes=870891520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=871350232 ingress_bytes=1097383936 link_bytes=1097383936 token_bytes=1742241752 weight_movement_bytes=226492416 total_nvlink_bytes=1968734168 weight_to_token_ratio=0.130001 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.391637299e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=839712768 dispatch_meta_bytes=458768 combine_payload_bytes=839712768 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=840171536 ingress_bytes=990707712 link_bytes=990707712 token_bytes=1679884304 weight_movement_bytes=150994944 total_nvlink_bytes=1830879248 weight_to_token_ratio=0.089884 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.256356831e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=799358976 dispatch_meta_bytes=451768 combine_payload_bytes=799358976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=799810744 ingress_bytes=874856448 link_bytes=874856448 token_bytes=1599169720 weight_movement_bytes=75497472 total_nvlink_bytes=1674667192 weight_to_token_ratio=0.047210 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=51840 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.109441121e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123696 payload_rows_remote=107312 dup_rows=7376 dup_rows_remote=7368 dedup_ratio=0.056274 router_tokens_per_expert=[8192, 0, 0, 0, 0, 0, 8192, 0, 652, 0, 0, 49, 0, 0, 7612, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5048, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8163, 1, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[15354, 16384, 16384, 16384, 10096, 16384, 16384, 16326] dup_rows_per_dst=[1272, 72, 8, 2, 0, 0, 6020, 2] combine_rows_per_src=[15354, 16384, 16384, 16384, 10096, 16384, 16384, 16326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130374 payload_rows_remote=114346 dup_rows=698 dup_rows_remote=698 dedup_ratio=0.005325 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 8133, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8014, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 305, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8081, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16272, 16384, 16028, 16384, 16376, 16162] dup_rows_per_dst=[16, 48, 4, 20, 0, 610, 0, 0] combine_rows_per_src=[16384, 16384, 16272, 16384, 16028, 16384, 16376, 16162] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102968 payload_rows_remote=86688 dup_rows=28104 dup_rows_remote=16348 dedup_ratio=0.214417 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 8127, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 669, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 7] payload_rows_per_dst=[16032, 16152, 62, 5300, 16376, 16384, 16382, 16280] dup_rows_per_dst=[60, 0, 0, 150, 14800, 1338, 0, 11756] combine_rows_per_src=[16032, 16152, 62, 5300, 16376, 16384, 16382, 16280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=879099904 dispatch_meta_bytes=458720 combine_payload_bytes=879099904 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=879558624 ingress_bytes=1030094848 link_bytes=1030094848 token_bytes=1758658528 weight_movement_bytes=150994944 total_nvlink_bytes=1909653472 weight_to_token_ratio=0.085858 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.306305264e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126410 payload_rows_remote=110026 dup_rows=4662 dup_rows_remote=3982 dedup_ratio=0.035568 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 2, 0, 8188, 0, 352, 0, 0, 0, 0, 681, 0, 0, 0, 0, 7662, 0, 8189, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 336, 2, 0, 0, 0, 0, 0, 2, 0, 0, 6418, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1464, 0, 0, 0, 0, 0, 2, 0, 0, 7678, 0, 0, 0, 0, 0, 0, 0, 0, 8160, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 1, 0, 8190, 0, 0] payload_rows_per_dst=[16384, 16384, 16354, 16384, 12840, 16378, 15360, 16326] dup_rows_per_dst=[4, 10, 1036, 680, 4, 2928, 0, 0] combine_rows_per_src=[16384, 16384, 16354, 16384, 12840, 16378, 15360, 16326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117376 payload_rows_remote=100992 dup_rows=13696 dup_rows_remote=12522 dedup_ratio=0.104492 router_tokens_per_expert=[8180, 0, 0, 0, 0, 0, 8186, 0, 157, 0, 0, 216, 0, 0, 8109, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 587, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 6670, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 727, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16302, 16384, 16360, 16384, 13342, 16374, 5998, 16232] dup_rows_per_dst=[662, 1174, 0, 11010, 0, 0, 850, 0] combine_rows_per_src=[16302, 16384, 16360, 16384, 13342, 16374, 5998, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=107634 payload_rows_remote=91250 dup_rows=23438 dup_rows_remote=23398 dedup_ratio=0.178818 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 12, 0, 28, 0, 0, 0, 0, 8191, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 933, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6927, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 249, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 28] payload_rows_per_dst=[16384, 16384, 16384, 16384, 13854, 6240, 16376, 5628] dup_rows_per_dst=[40, 0, 19638, 1866, 0, 70, 1824, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 13854, 6240, 16376, 5628] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123606 payload_rows_remote=107226 dup_rows=7466 dup_rows_remote=4586 dedup_ratio=0.056961 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 399, 0, 0, 1, 1, 1027, 0, 0, 0, 0, 7861, 0, 8190, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1186, 0, 0, 0, 0, 0, 0, 1, 0, 0, 4830, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1440, 0, 0, 0, 0, 0, 0, 0, 0, 7830, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8191, 0, 1, 0, 1, 1, 0, 0, 0, 8191, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 9662, 16380, 15660, 16370] dup_rows_per_dst=[10, 0, 2196, 2374, 2, 2880, 0, 4] combine_rows_per_src=[16384, 16382, 16384, 16384, 9662, 16380, 15660, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=936722432 dispatch_meta_bytes=460176 combine_payload_bytes=936722432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937182608 ingress_bytes=1012219904 link_bytes=1012219904 token_bytes=1873905040 weight_movement_bytes=75497472 total_nvlink_bytes=1949402512 weight_to_token_ratio=0.040289 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=54528 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283637319e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=710148096 dispatch_meta_bytes=412144 combine_payload_bytes=710148096 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=710560240 ingress_bytes=1087635456 link_bytes=1087635456 token_bytes=1420708336 weight_movement_bytes=377487360 total_nvlink_bytes=1798195696 weight_to_token_ratio=0.265704 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=59648 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.379274854e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123740 payload_rows_remote=107416 dup_rows=7332 dup_rows_remote=7332 dedup_ratio=0.055939 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 77, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 8022, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7941, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 528, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 0, 0, 4, 0, 0, 7940, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 10096, 16384, 15904, 16384, 16324, 15880] dup_rows_per_dst=[70, 150, 6, 124, 5926, 1056, 0, 0] combine_rows_per_src=[16384, 16384, 10096, 16384, 15904, 16384, 16324, 15880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=901332992 dispatch_meta_bytes=456032 combine_payload_bytes=901332992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=901789024 ingress_bytes=976830464 link_bytes=976830464 token_bytes=1803122016 weight_movement_bytes=75497472 total_nvlink_bytes=1878619488 weight_to_token_ratio=0.041870 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=60032 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.238758528e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=827326464 dispatch_meta_bytes=454056 combine_payload_bytes=827326464 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=827780520 ingress_bytes=978321408 link_bytes=978321408 token_bytes=1655106984 weight_movement_bytes=150994944 total_nvlink_bytes=1806101928 weight_to_token_ratio=0.091230 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.045956 gemm_active_groups=10 gemm_peak_group_rows=57216 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.240649254e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=747520000 dispatch_meta_bytes=458592 combine_payload_bytes=747520000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=747978592 ingress_bytes=823017472 link_bytes=823017472 token_bytes=1495498592 weight_movement_bytes=75497472 total_nvlink_bytes=1570996064 weight_to_token_ratio=0.050483 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.043702003e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=878395392 dispatch_meta_bytes=447248 combine_payload_bytes=878395392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=878842640 ingress_bytes=953892864 link_bytes=953892864 token_bytes=1757238032 weight_movement_bytes=75497472 total_nvlink_bytes=1832735504 weight_to_token_ratio=0.042964 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=60288 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.209670422e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=879951872 dispatch_meta_bytes=458992 combine_payload_bytes=879951872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=880410864 ingress_bytes=955449344 link_bytes=955449344 token_bytes=1760362736 weight_movement_bytes=75497472 total_nvlink_bytes=1835860208 weight_to_token_ratio=0.042887 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.211644258e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124522 payload_rows_remote=108214 dup_rows=6550 dup_rows_remote=6476 dedup_ratio=0.049973 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 30, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7, 0, 34, 0, 0, 0, 0, 8150, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2926, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 274, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5] payload_rows_per_dst=[16384, 16384, 16308, 16384, 10006, 16384, 16382, 16290] dup_rows_per_dst=[62, 8, 74, 5852, 0, 548, 0, 6] combine_rows_per_src=[16384, 16384, 16308, 16384, 10006, 16384, 16382, 16290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119638 payload_rows_remote=106348 dup_rows=11434 dup_rows_remote=6054 dedup_ratio=0.087234 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3064, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 1914, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3012, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 8064, 16384, 16384, 13290, 16364, 16384, 16384] dup_rows_per_dst=[48, 1796, 3832, 0, 5380, 0, 378, 0] combine_rows_per_src=[16384, 8064, 16384, 16384, 13290, 16364, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99114 payload_rows_remote=82730 dup_rows=31958 dup_rows_remote=6076 dedup_ratio=0.243820 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3406, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1812, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2877, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 71, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[15982, 6812, 3624, 16164, 16384, 16384, 7380, 16384] dup_rows_per_dst=[48, 0, 0, 0, 5754, 140, 134, 25882] combine_rows_per_src=[15982, 6812, 3624, 16164, 16384, 16384, 7380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114800 payload_rows_remote=98416 dup_rows=16272 dup_rows_remote=16108 dedup_ratio=0.124146 router_tokens_per_expert=[0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 2723, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 2932, 0, 2, 2, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 75, 0, 2, 0, 0, 0, 0, 8182, 0, 0, 0, 8168, 1, 0, 0, 0, 4, 0, 0, 11, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2375, 0, 0, 0, 8190, 8189, 0, 0, 0, 0, 0, 0, 8190, 8188, 0, 0, 2] payload_rows_per_dst=[16380, 16374, 16382, 16384, 154, 16362, 16380, 16384] dup_rows_per_dst=[6, 5474, 5868, 164, 0, 10, 4750, 0] combine_rows_per_src=[16380, 16374, 16382, 16384, 154, 16362, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114684 payload_rows_remote=98300 dup_rows=16388 dup_rows_remote=16388 dedup_ratio=0.125031 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4625, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1463, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2021, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 0] dup_rows_per_dst=[0, 9250, 2926, 0, 4042, 0, 170, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16380, 16384, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=886489088 dispatch_meta_bytes=458760 combine_payload_bytes=886489088 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=886947848 ingress_bytes=1112981504 link_bytes=1112981504 token_bytes=1773436936 weight_movement_bytes=226492416 total_nvlink_bytes=1999929352 weight_to_token_ratio=0.127714 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=53248 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.411417211e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130216 payload_rows_remote=113832 dup_rows=856 dup_rows_remote=838 dedup_ratio=0.006531 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 45, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8129, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 362, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 2, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1] payload_rows_per_dst=[16384, 16384, 16258, 16384, 15670, 16384, 16384, 16368] dup_rows_per_dst=[90, 18, 0, 18, 0, 724, 4, 2] combine_rows_per_src=[16384, 16384, 16258, 16384, 15670, 16384, 16384, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=871202816 dispatch_meta_bytes=449608 combine_payload_bytes=871202816 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=871652424 ingress_bytes=1022197760 link_bytes=1022197760 token_bytes=1742855240 weight_movement_bytes=150994944 total_nvlink_bytes=1893850184 weight_to_token_ratio=0.086637 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=30976 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.296290645e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121014 payload_rows_remote=104648 dup_rows=10058 dup_rows_remote=10058 dedup_ratio=0.076736 router_tokens_per_expert=[1, 0, 0, 0, 0, 0, 0, 0, 0, 122, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 1, 0, 0, 0, 3190, 0, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1857, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 8190, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 1, 1766, 0, 1, 0, 0, 0, 0, 8189, 0, 0, 0, 8181, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1253, 0, 0, 0, 8189, 8191, 0, 0, 0, 0, 0, 0, 8167, 8190, 0, 0, 1] payload_rows_per_dst=[16382, 6398, 16382, 16384, 16384, 16366, 16382, 16336] dup_rows_per_dst=[248, 4, 3716, 56, 3530, 0, 2504, 0] combine_rows_per_src=[16382, 6398, 16382, 16384, 16384, 16366, 16382, 16336] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119064 payload_rows_remote=102680 dup_rows=12008 dup_rows_remote=11842 dedup_ratio=0.091614 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 25, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5732, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 1107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1256, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 83, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 11464, 9328, 16384, 16380, 16356, 16384, 16384] dup_rows_per_dst=[50, 0, 1772, 0, 2512, 0, 166, 7508] combine_rows_per_src=[16384, 11464, 9328, 16384, 16380, 16356, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=677724160 dispatch_meta_bytes=355224 combine_payload_bytes=677724160 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=678079384 ingress_bytes=1055211520 link_bytes=1055211520 token_bytes=1355803544 weight_movement_bytes=377487360 total_nvlink_bytes=1733290904 weight_to_token_ratio=0.278423 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=48896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.338156740e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=806223872 dispatch_meta_bytes=458096 combine_payload_bytes=806223872 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=806681968 ingress_bytes=957218816 link_bytes=957218816 token_bytes=1612905840 weight_movement_bytes=150994944 total_nvlink_bytes=1763900784 weight_to_token_ratio=0.093617 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.213888197e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=805273600 dispatch_meta_bytes=458752 combine_payload_bytes=805273600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=805732352 ingress_bytes=880771072 link_bytes=880771072 token_bytes=1611005952 weight_movement_bytes=75497472 total_nvlink_bytes=1686503424 weight_to_token_ratio=0.046864 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116941697e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=932511744 dispatch_meta_bytes=458680 combine_payload_bytes=932511744 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=932970424 ingress_bytes=1008009216 link_bytes=1008009216 token_bytes=1865482168 weight_movement_bytes=75497472 total_nvlink_bytes=1940979640 weight_to_token_ratio=0.040471 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278297574e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=857276416 dispatch_meta_bytes=458824 combine_payload_bytes=857276416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=857735240 ingress_bytes=932773888 link_bytes=932773888 token_bytes=1715011656 weight_movement_bytes=75497472 total_nvlink_bytes=1790509128 weight_to_token_ratio=0.044022 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.182888588e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=841154560 dispatch_meta_bytes=458088 combine_payload_bytes=841154560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=841612648 ingress_bytes=916652032 link_bytes=916652032 token_bytes=1682767208 weight_movement_bytes=75497472 total_nvlink_bytes=1758264680 weight_to_token_ratio=0.044865 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=61056 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.162443805e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126972 payload_rows_remote=110588 dup_rows=4100 dup_rows_remote=3784 dedup_ratio=0.031281 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1682, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6228, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 12874, 16368, 16384, 15810] dup_rows_per_dst=[8, 3364, 316, 0, 154, 0, 258, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 12874, 16368, 16384, 15810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128454 payload_rows_remote=112070 dup_rows=2618 dup_rows_remote=6 dedup_ratio=0.019974 router_tokens_per_expert=[0, 0, 8188, 0, 0, 8192, 0, 0, 0, 5, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7630, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7452, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 27, 0, 0, 6, 1273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 15260, 16380, 14904, 16384, 16384, 16376, 16384] dup_rows_per_dst=[6, 0, 0, 0, 2612, 0, 0, 0] combine_rows_per_src=[16382, 15260, 16380, 14904, 16384, 16384, 16376, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108318 payload_rows_remote=91934 dup_rows=22754 dup_rows_remote=18712 dedup_ratio=0.173599 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7021, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6623, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 29, 2689, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[10596, 14042, 1760, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[6, 0, 0, 13242, 5464, 0, 0, 4042] combine_rows_per_src=[10596, 14042, 1760, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109390 payload_rows_remote=107654 dup_rows=21682 dup_rows_remote=21678 dedup_ratio=0.165421 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8188, 0, 0, 2, 12, 0, 0, 0, 8188, 1474, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 6100, 0, 0, 0, 8044, 0, 0, 0, 0, 8103, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 868, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 5578, 3794, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2, 2, 0, 0, 12, 0, 2, 0, 0, 0, 0, 6948, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12204, 16378, 1736, 16020, 16384, 16384, 13900] dup_rows_per_dst=[2968, 0, 14314, 4, 4356, 2, 34, 4] combine_rows_per_src=[16384, 12204, 16378, 1736, 16020, 16384, 16384, 13900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117848 payload_rows_remote=101464 dup_rows=13224 dup_rows_remote=13142 dedup_ratio=0.100891 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 17, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6533, 0, 0, 0, 8133, 0, 0, 0, 0, 8192, 0, 40, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6620, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3232, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16270, 16384, 13240, 6464, 16380, 16342, 16384] dup_rows_per_dst=[82, 13062, 80, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16270, 16384, 13240, 6464, 16380, 16342, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=905936896 dispatch_meta_bytes=457488 combine_payload_bytes=905936896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=906394384 ingress_bytes=981434368 link_bytes=981434368 token_bytes=1812331280 weight_movement_bytes=75497472 total_nvlink_bytes=1887828752 weight_to_token_ratio=0.041658 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=53632 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.244596926e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114646 payload_rows_remote=98262 dup_rows=16426 dup_rows_remote=3506 dedup_ratio=0.125320 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6458, 0, 0, 2, 0, 0, 0, 0, 0, 13, 0, 682, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 733, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 325, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16342, 16384, 0] dup_rows_per_dst=[0, 12920, 1390, 0, 1466, 0, 650, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16342, 16384, 0] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=918077440 dispatch_meta_bytes=448304 combine_payload_bytes=918077440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=918525744 ingress_bytes=993574912 link_bytes=993574912 token_bytes=1836603184 weight_movement_bytes=75497472 total_nvlink_bytes=1912100656 weight_to_token_ratio=0.041107 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=33536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.259992845e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111192 payload_rows_remote=94810 dup_rows=19880 dup_rows_remote=19876 dedup_ratio=0.151672 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8189, 0, 0, 2, 72, 0, 0, 0, 8189, 1270, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 6535, 0, 0, 0, 8132, 0, 0, 0, 0, 8121, 0, 55, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2433, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2170, 4739, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 8, 0, 1, 0, 0, 0, 0, 7409, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 13074, 6476, 11290, 16380, 16382, 16384, 14822] dup_rows_per_dst=[2682, 2, 92, 3398, 13680, 4, 22, 0] combine_rows_per_src=[16384, 13074, 6476, 11290, 16380, 16382, 16384, 14822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113270 payload_rows_remote=96892 dup_rows=17802 dup_rows_remote=17802 dedup_ratio=0.135818 router_tokens_per_expert=[0, 0, 8189, 0, 0, 8192, 0, 0, 0, 17, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7477, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5806, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 1, 3052, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 14954, 24, 16380, 16384, 16384, 16378, 16384] dup_rows_per_dst=[28, 0, 0, 11606, 6168, 0, 0, 0] combine_rows_per_src=[16382, 14954, 24, 16380, 16384, 16384, 16378, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=753123328 dispatch_meta_bytes=442584 combine_payload_bytes=753123328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=753565912 ingress_bytes=828620800 link_bytes=828620800 token_bytes=1506689240 weight_movement_bytes=75497472 total_nvlink_bytes=1582186712 weight_to_token_ratio=0.050108 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=63616 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.050807812e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=881901568 dispatch_meta_bytes=517328 combine_payload_bytes=881901568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=882418896 ingress_bytes=957399040 link_bytes=957399040 token_bytes=1764320464 weight_movement_bytes=75497472 total_nvlink_bytes=1839817936 weight_to_token_ratio=0.042791 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=44288 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.214116747e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=831193088 dispatch_meta_bytes=458424 combine_payload_bytes=831193088 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=831651512 ingress_bytes=1133182976 link_bytes=1133182976 token_bytes=1662844600 weight_movement_bytes=301989888 total_nvlink_bytes=1964834488 weight_to_token_ratio=0.181610 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.437035521e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=776683520 dispatch_meta_bytes=458744 combine_payload_bytes=776683520 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=777142264 ingress_bytes=852180992 link_bytes=852180992 token_bytes=1553825784 weight_movement_bytes=75497472 total_nvlink_bytes=1629323256 weight_to_token_ratio=0.048588 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080685451e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=804962304 dispatch_meta_bytes=407072 combine_payload_bytes=804962304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=805369376 ingress_bytes=880459776 link_bytes=880459776 token_bytes=1610331680 weight_movement_bytes=75497472 total_nvlink_bytes=1685829152 weight_to_token_ratio=0.046883 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=34688 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116546930e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=793739264 dispatch_meta_bytes=458776 combine_payload_bytes=793739264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=794198040 ingress_bytes=869236736 link_bytes=869236736 token_bytes=1587937304 weight_movement_bytes=75497472 total_nvlink_bytes=1663434776 weight_to_token_ratio=0.047544 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.102314536e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114658 payload_rows_remote=98274 dup_rows=16414 dup_rows_remote=18 dedup_ratio=0.125229 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 4, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5760, 0, 0, 0, 8190, 0, 0, 0, 0, 8192, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7951, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 25, 2630, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 11520, 16384, 15902, 5316, 16384, 16384, 16384] dup_rows_per_dst=[8, 0, 16396, 0, 10, 0, 0, 0] combine_rows_per_src=[16384, 11520, 16384, 15902, 5316, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127702 payload_rows_remote=113670 dup_rows=3370 dup_rows_remote=3370 dedup_ratio=0.025711 router_tokens_per_expert=[0, 0, 8192, 1, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 1, 0, 1679, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 2, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7693, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 14032, 16382, 15386, 16366] dup_rows_per_dst=[16, 0, 3340, 12, 0, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 14032, 16382, 15386, 16366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116166 payload_rows_remote=99782 dup_rows=14906 dup_rows_remote=14822 dedup_ratio=0.113724 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8052, 0, 0, 0, 0, 0, 0, 5, 0, 1780, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7576, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7163, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16300, 16384, 3560, 16384, 16260, 16384, 14510, 16384] dup_rows_per_dst=[0, 0, 10, 0, 11022, 0, 3790, 84] combine_rows_per_src=[16300, 16384, 3560, 16384, 16260, 16384, 14510, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122598 payload_rows_remote=109064 dup_rows=8474 dup_rows_remote=5392 dedup_ratio=0.064651 router_tokens_per_expert=[0, 0, 8153, 21, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 1589, 0, 0, 0, 0, 0, 0, 0, 0, 2, 6717, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 2, 13, 0, 0, 0, 6834, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 4653, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4963, 0, 0, 0, 8183, 0, 0, 0, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16348, 16382, 16054, 13534, 13672, 16360, 13878, 16370] dup_rows_per_dst=[4, 2, 0, 3082, 28, 0, 5354, 4] combine_rows_per_src=[16348, 16382, 16054, 13534, 13672, 16360, 13878, 16370] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=805060608 dispatch_meta_bytes=393168 combine_payload_bytes=805060608 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=805453776 ingress_bytes=1107050496 link_bytes=1107050496 token_bytes=1610514384 weight_movement_bytes=301989888 total_nvlink_bytes=1912504272 weight_to_token_ratio=0.187511 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=32768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.403895858e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124884 payload_rows_remote=108500 dup_rows=6188 dup_rows_remote=6128 dedup_ratio=0.047211 router_tokens_per_expert=[0, 0, 8192, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7993, 0, 0, 0, 0, 0, 0, 2, 0, 3223, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 10, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 1, 0, 0, 0, 7428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5899, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16374, 16384, 16384, 14824, 16384, 11766, 16384] dup_rows_per_dst=[60, 52, 6014, 0, 2, 32, 28, 0] combine_rows_per_src=[16384, 16374, 16384, 16384, 14824, 16384, 11766, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124488 payload_rows_remote=108106 dup_rows=6584 dup_rows_remote=6584 dedup_ratio=0.050232 router_tokens_per_expert=[0, 0, 8183, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 7707, 0, 0, 0, 2, 2, 0, 2, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 1, 0, 0, 0, 1, 7126, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 1, 0, 0, 0, 6461, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 6370, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5048, 0, 0, 0, 8190, 0, 0, 1, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16368, 16384, 15424, 14290, 12926, 16382, 16330, 16384] dup_rows_per_dst=[2, 4, 2, 64, 2, 0, 6506, 4] combine_rows_per_src=[16368, 16384, 15424, 14290, 12926, 16382, 16330, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=931184640 dispatch_meta_bytes=468160 combine_payload_bytes=931184640 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=931652800 ingress_bytes=1157677056 link_bytes=1157677056 token_bytes=1862837440 weight_movement_bytes=226492416 total_nvlink_bytes=2089329856 weight_to_token_ratio=0.121585 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=55680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.468097462e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119572 payload_rows_remote=105272 dup_rows=11500 dup_rows_remote=11500 dedup_ratio=0.087738 router_tokens_per_expert=[0, 0, 8192, 3, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 1286, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 14, 0, 0, 0, 3, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 7955, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7150, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16366, 7348, 16384, 16050, 16380, 14300, 16360] dup_rows_per_dst=[30, 0, 2152, 1778, 7540, 0, 0, 0] combine_rows_per_src=[16384, 16366, 7348, 16384, 16050, 16380, 14300, 16360] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115666 payload_rows_remote=100350 dup_rows=15406 dup_rows_remote=10038 dedup_ratio=0.117538 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 7, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7166, 0, 0, 0, 8174, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6493, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 1, 2713, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15316, 16384, 12986, 5444, 16384, 16384, 16384] dup_rows_per_dst=[12, 5368, 10020, 0, 6, 0, 0, 0] combine_rows_per_src=[16384, 15316, 16384, 12986, 5444, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=817414144 dispatch_meta_bytes=458416 combine_payload_bytes=817414144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=817872560 ingress_bytes=892911616 link_bytes=892911616 token_bytes=1635286704 weight_movement_bytes=75497472 total_nvlink_bytes=1710784176 weight_to_token_ratio=0.046168 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.132337616e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=893452288 dispatch_meta_bytes=457824 combine_payload_bytes=893452288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=893910112 ingress_bytes=968949760 link_bytes=968949760 token_bytes=1787362400 weight_movement_bytes=75497472 total_nvlink_bytes=1862859872 weight_to_token_ratio=0.042240 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=63104 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.228764686e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=888832000 dispatch_meta_bytes=458512 combine_payload_bytes=888832000 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=889290512 ingress_bytes=1039826944 link_bytes=1039826944 token_bytes=1778122512 weight_movement_bytes=150994944 total_nvlink_bytes=1929117456 weight_to_token_ratio=0.084918 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.318646932e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=885604352 dispatch_meta_bytes=458760 combine_payload_bytes=885604352 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=886063112 ingress_bytes=1036599296 link_bytes=1036599296 token_bytes=1771667464 weight_movement_bytes=150994944 total_nvlink_bytes=1922662408 weight_to_token_ratio=0.085228 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.314553820e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=822067200 dispatch_meta_bytes=441552 combine_payload_bytes=822067200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=822508752 ingress_bytes=897564672 link_bytes=897564672 token_bytes=1644575952 weight_movement_bytes=75497472 total_nvlink_bytes=1720073424 weight_to_token_ratio=0.045907 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=54272 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.138238346e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=862388224 dispatch_meta_bytes=467088 combine_payload_bytes=862388224 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=862855312 ingress_bytes=1088880640 link_bytes=1088880640 token_bytes=1725243536 weight_movement_bytes=226492416 total_nvlink_bytes=1951735952 weight_to_token_ratio=0.131281 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=52608 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.380853923e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121332 payload_rows_remote=104948 dup_rows=9740 dup_rows_remote=42 dedup_ratio=0.074310 router_tokens_per_expert=[0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7664, 0, 0, 0, 0, 0, 0, 233, 0, 5144, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 2, 0, 0, 0, 4423, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 16, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7096, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16382, 16384, 16384, 8846, 16384, 14194, 16374] dup_rows_per_dst=[0, 0, 9698, 8, 4, 0, 30, 0] combine_rows_per_src=[16384, 16382, 16384, 16384, 8846, 16384, 14194, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128396 payload_rows_remote=112012 dup_rows=2676 dup_rows_remote=2676 dedup_ratio=0.020416 router_tokens_per_expert=[0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 7, 0, 0, 0, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1456, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 208, 0, 0, 0, 0, 6696] payload_rows_per_dst=[16384, 16384, 16384, 16344, 16384, 16342, 16384, 13790] dup_rows_per_dst=[0, 0, 46, 2594, 0, 0, 18, 18] combine_rows_per_src=[16384, 16384, 16384, 16344, 16384, 16342, 16384, 13790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115428 payload_rows_remote=99594 dup_rows=15644 dup_rows_remote=4384 dedup_ratio=0.119354 router_tokens_per_expert=[0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2221, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 107, 0, 0, 0, 0, 5961] payload_rows_per_dst=[1396, 16384, 16384, 16296, 16376, 16374, 16384, 15834] dup_rows_per_dst=[0, 0, 0, 4376, 0, 0, 8, 11260] combine_rows_per_src=[1396, 16384, 16384, 16296, 16376, 16374, 16384, 15834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116126 payload_rows_remote=99752 dup_rows=14946 dup_rows_remote=3596 dedup_ratio=0.114029 router_tokens_per_expert=[0, 0, 0, 0, 8142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 810, 0, 0, 0, 0, 1, 11, 0, 0, 8190, 0, 1436, 0, 2, 0, 8188, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 0, 0, 43, 0, 5669, 0, 0, 0, 2, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 2, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 340, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 0, 0, 0, 8121] payload_rows_per_dst=[1622, 16384, 16384, 16374, 16378, 16370, 16372, 16242] dup_rows_per_dst=[0, 2892, 4, 11350, 12, 0, 680, 8] combine_rows_per_src=[1622, 16384, 16384, 16374, 16378, 16370, 16372, 16242] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=859734016 dispatch_meta_bytes=419960 combine_payload_bytes=859734016 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=860153976 ingress_bytes=1161723904 link_bytes=1161723904 token_bytes=1719887992 weight_movement_bytes=301989888 total_nvlink_bytes=2021877880 weight_to_token_ratio=0.175587 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=51200 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.473229435e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117530 payload_rows_remote=101146 dup_rows=13542 dup_rows_remote=13540 dedup_ratio=0.103317 router_tokens_per_expert=[2, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1479, 0, 0, 0, 0, 0, 1, 0, 4, 8190, 0, 919, 0, 2, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 1, 0, 3, 0, 5586, 0, 0, 0, 2, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 245, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8139] payload_rows_per_dst=[2962, 16384, 16382, 16384, 16370, 16384, 16384, 16280] dup_rows_per_dst=[0, 1846, 0, 11188, 16, 2, 490, 0] combine_rows_per_src=[2962, 16384, 16382, 16384, 16370, 16384, 16384, 16280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108512 payload_rows_remote=92128 dup_rows=22560 dup_rows_remote=708 dedup_ratio=0.172119 router_tokens_per_expert=[0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8116, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1460, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8092, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 6903] payload_rows_per_dst=[16384, 11454, 16194, 2920, 16384, 16184, 15182, 13810] dup_rows_per_dst=[21852, 192, 0, 0, 0, 480, 0, 36] combine_rows_per_src=[16384, 11454, 16194, 2920, 16384, 16184, 15182, 13810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=917602304 dispatch_meta_bytes=458752 combine_payload_bytes=917602304 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=918061056 ingress_bytes=993099776 link_bytes=993099776 token_bytes=1835663360 weight_movement_bytes=75497472 total_nvlink_bytes=1911160832 weight_to_token_ratio=0.041128 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.259390305e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128256 payload_rows_remote=111874 dup_rows=2816 dup_rows_remote=2812 dedup_ratio=0.021484 router_tokens_per_expert=[0, 0, 0, 0, 8003, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 38, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1283, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 294, 0, 0, 0, 0, 6818] payload_rows_per_dst=[16382, 16384, 16384, 16154, 16384, 16222, 16382, 13964] dup_rows_per_dst=[0, 0, 132, 2418, 0, 0, 4, 262] combine_rows_per_src=[16382, 16384, 16384, 16154, 16384, 16222, 16382, 13964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129108 payload_rows_remote=112750 dup_rows=1964 dup_rows_remote=1964 dedup_ratio=0.014984 router_tokens_per_expert=[0, 0, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8152, 0, 0, 0, 0, 0, 0, 3, 0, 1004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 16, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 7916, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7496, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16358, 16384, 16384, 15864, 16384, 14992, 16358] dup_rows_per_dst=[24, 0, 1934, 0, 0, 0, 6, 0] combine_rows_per_src=[16384, 16358, 16384, 16384, 15864, 16384, 14992, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=815874048 dispatch_meta_bytes=415912 combine_payload_bytes=815874048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=816289960 ingress_bytes=891371520 link_bytes=891371520 token_bytes=1632164008 weight_movement_bytes=75497472 total_nvlink_bytes=1707661480 weight_to_token_ratio=0.046256 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=57088 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.130384557e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=817168384 dispatch_meta_bytes=413392 combine_payload_bytes=817168384 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=817581776 ingress_bytes=892665856 link_bytes=892665856 token_bytes=1634750160 weight_movement_bytes=75497472 total_nvlink_bytes=1710247632 weight_to_token_ratio=0.046183 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=45952 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.132025958e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=828588032 dispatch_meta_bytes=458744 combine_payload_bytes=828588032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=829046776 ingress_bytes=904085504 link_bytes=904085504 token_bytes=1657634808 weight_movement_bytes=75497472 total_nvlink_bytes=1733132280 weight_to_token_ratio=0.045545 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.146507679e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=754712576 dispatch_meta_bytes=371344 combine_payload_bytes=754712576 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=755083920 ingress_bytes=981204992 link_bytes=981204992 token_bytes=1509796496 weight_movement_bytes=226492416 total_nvlink_bytes=1736288912 weight_to_token_ratio=0.150015 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=43904 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.244306045e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=916471808 dispatch_meta_bytes=458744 combine_payload_bytes=916471808 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=916930552 ingress_bytes=1142964224 link_bytes=1142964224 token_bytes=1833402360 weight_movement_bytes=226492416 total_nvlink_bytes=2059894776 weight_to_token_ratio=0.123537 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.449439520e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=923648000 dispatch_meta_bytes=458856 combine_payload_bytes=923648000 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=924106856 ingress_bytes=1074642944 link_bytes=1074642944 token_bytes=1847754856 weight_movement_bytes=150994944 total_nvlink_bytes=1998749800 weight_to_token_ratio=0.081718 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.362798520e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125548 payload_rows_remote=109164 dup_rows=5524 dup_rows_remote=5514 dedup_ratio=0.042145 router_tokens_per_expert=[0, 0, 0, 0, 8101, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8171, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 660, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 67, 0, 0, 0, 0, 7568] payload_rows_per_dst=[16382, 16384, 16384, 12046, 16384, 16376, 16384, 15208] dup_rows_per_dst=[5192, 0, 10, 244, 0, 0, 16, 62] combine_rows_per_src=[16382, 16384, 16384, 12046, 16384, 16376, 16384, 15208] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129850 payload_rows_remote=113466 dup_rows=1222 dup_rows_remote=1222 dedup_ratio=0.009323 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 7931, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7471, 0, 0, 0, 0, 591, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 8192, 0] payload_rows_per_dst=[15876, 16384, 16372, 15684, 16384, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 440, 0, 0, 0, 782] combine_rows_per_src=[15876, 16384, 16372, 15684, 16384, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129344 payload_rows_remote=112960 dup_rows=1728 dup_rows_remote=160 dedup_ratio=0.013184 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8113, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6340, 0, 0, 0, 0, 1147, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 784, 0, 8192, 0] payload_rows_per_dst=[16224, 16384, 16384, 14816, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 158, 0, 0, 2, 1568] combine_rows_per_src=[16224, 16384, 16384, 14816, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130882 payload_rows_remote=114498 dup_rows=190 dup_rows_remote=164 dedup_ratio=0.001450 router_tokens_per_expert=[0, 8189, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 1, 8153, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 2, 0, 0, 0, 2, 0, 6567, 0, 1623, 0, 0, 7, 2, 0, 2, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 8189, 0, 0, 0, 0, 0, 0, 8102, 0, 0, 0, 43, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 78, 0, 8188, 0] payload_rows_per_dst=[16380, 16308, 16374, 16384, 16382, 16294, 16380, 16380] dup_rows_per_dst=[2, 0, 4, 26, 4, 0, 0, 154] combine_rows_per_src=[16380, 16308, 16374, 16384, 16382, 16294, 16380, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=894271488 dispatch_meta_bytes=458712 combine_payload_bytes=894271488 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=894730200 ingress_bytes=969768960 link_bytes=969768960 token_bytes=1789001688 weight_movement_bytes=75497472 total_nvlink_bytes=1864499160 weight_to_token_ratio=0.042201 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.229803547e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130958 payload_rows_remote=114610 dup_rows=114 dup_rows_remote=108 dedup_ratio=0.000870 router_tokens_per_expert=[0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 2, 1, 0, 0, 1, 0, 6515, 0, 1674, 0, 0, 1, 2, 0, 0, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 8187, 0, 0, 0, 0, 0, 0, 8153, 0, 0, 0, 23, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 8190, 0] payload_rows_per_dst=[16380, 16328, 16378, 16384, 16378, 16348, 16382, 16380] dup_rows_per_dst=[0, 0, 4, 8, 4, 6, 4, 88] combine_rows_per_src=[16380, 16328, 16378, 16384, 16378, 16348, 16382, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=929513472 dispatch_meta_bytes=458752 combine_payload_bytes=929513472 prefetch_bytes=0 reduce_bytes=0 egress_bytes=929972224 ingress_bytes=929513472 link_bytes=929972224 token_bytes=1859485696 weight_movement_bytes=0 total_nvlink_bytes=1859485696 weight_to_token_ratio=0.000000 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.179335684e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129612 payload_rows_remote=113228 dup_rows=1460 dup_rows_remote=1460 dedup_ratio=0.011139 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 7834, 0, 0, 0, 8191, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6736, 0, 0, 0, 0, 1437, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 377, 0, 8192, 0] payload_rows_per_dst=[15680, 16384, 16376, 15640, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 706, 0, 0, 0, 754] combine_rows_per_src=[15680, 16384, 16376, 15640, 16384, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117884 payload_rows_remote=101522 dup_rows=13188 dup_rows_remote=11200 dedup_ratio=0.100616 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8138, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4969, 0, 0, 0, 0, 1148, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2128, 0, 8192, 0] payload_rows_per_dst=[16362, 16384, 16384, 15296, 16384, 16384, 16384, 4306] dup_rows_per_dst=[1988, 132, 28, 10906, 0, 116, 0, 18] combine_rows_per_src=[16362, 16384, 16384, 15296, 16384, 16384, 16384, 4306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=925368320 dispatch_meta_bytes=452480 combine_payload_bytes=925368320 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=925820800 ingress_bytes=1302855680 link_bytes=1302855680 token_bytes=1851189120 weight_movement_bytes=377487360 total_nvlink_bytes=2228676480 weight_to_token_ratio=0.203916 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=57472 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.652204393e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=937967616 dispatch_meta_bytes=458648 combine_payload_bytes=937967616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938426264 ingress_bytes=1013465088 link_bytes=1013465088 token_bytes=1876393880 weight_movement_bytes=75497472 total_nvlink_bytes=1951891352 weight_to_token_ratio=0.040235 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=47232 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285216388e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114698 payload_rows_remote=98314 dup_rows=16374 dup_rows_remote=16374 dedup_ratio=0.124924 router_tokens_per_expert=[0, 0, 0, 0, 8066, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 6861] payload_rows_per_dst=[16384, 16384, 16384, 2384, 16384, 16338, 16384, 14056] dup_rows_per_dst=[16132, 0, 0, 0, 0, 0, 0, 242] combine_rows_per_src=[16384, 16384, 16384, 2384, 16384, 16338, 16384, 14056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[4, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=938885120 dispatch_meta_bytes=458872 combine_payload_bytes=938885120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939343992 ingress_bytes=1014382592 link_bytes=1014382592 token_bytes=1878229112 weight_movement_bytes=75497472 total_nvlink_bytes=1953726584 weight_to_token_ratio=0.040196 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286379912e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=927563776 dispatch_meta_bytes=458752 combine_payload_bytes=927563776 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=928022528 ingress_bytes=1078558720 link_bytes=1078558720 token_bytes=1855586304 weight_movement_bytes=150994944 total_nvlink_bytes=2006581248 weight_to_token_ratio=0.081373 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.367764276e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=831668224 dispatch_meta_bytes=450888 combine_payload_bytes=831668224 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=832119112 ingress_bytes=1058160640 link_bytes=1058160640 token_bytes=1663787336 weight_movement_bytes=226492416 total_nvlink_bytes=1890279752 weight_to_token_ratio=0.136131 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=64512 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.341896639e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=805388288 dispatch_meta_bytes=458752 combine_payload_bytes=805388288 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=805847040 ingress_bytes=1031880704 link_bytes=1031880704 token_bytes=1611235328 weight_movement_bytes=226492416 total_nvlink_bytes=1837727744 weight_to_token_ratio=0.140571 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.308569981e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122884 payload_rows_remote=106500 dup_rows=8188 dup_rows_remote=8188 dedup_ratio=0.062469 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8143, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3685, 0, 0, 0, 0, 509, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4047, 0, 8192, 0] payload_rows_per_dst=[16286, 16384, 16384, 8294, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 94, 0, 0, 0, 8094] combine_rows_per_src=[16286, 16384, 16384, 8294, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125010 payload_rows_remote=108626 dup_rows=6062 dup_rows_remote=0 dedup_ratio=0.046249 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6187, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 3023, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8125, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7231, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 12378, 16384, 16250, 16384, 14462] dup_rows_per_dst=[0, 0, 0, 0, 6062, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 12378, 16384, 16250, 16384, 14462] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112314 payload_rows_remote=96424 dup_rows=18758 dup_rows_remote=5424 dedup_ratio=0.143112 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 7267, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2060, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8142, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7127, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16384, 2, 14536, 16384, 16352, 16384, 15890] dup_rows_per_dst=[0, 0, 0, 0, 4120, 1302, 2, 13334] combine_rows_per_src=[16382, 16384, 2, 14536, 16384, 16352, 16384, 15890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113904 payload_rows_remote=97528 dup_rows=17168 dup_rows_remote=11704 dedup_ratio=0.130981 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 181, 0, 0, 0, 0, 7138, 0, 2720, 0, 0, 0, 0, 0, 0, 1061, 0, 0, 0, 1, 0, 6368, 0, 2, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 11, 1, 7799, 0, 0, 0, 8188, 0, 0, 0, 967, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 2, 2, 6429, 2, 95, 0, 8178, 0, 0, 2, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16380, 2856, 16376, 15212, 15626, 14692, 16382] dup_rows_per_dst=[2, 4, 0, 5464, 11416, 0, 102, 180] combine_rows_per_src=[16380, 16380, 2856, 16376, 15212, 15626, 14692, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=872448000 dispatch_meta_bytes=458752 combine_payload_bytes=872448000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=872906752 ingress_bytes=947945472 link_bytes=947945472 token_bytes=1745354752 weight_movement_bytes=75497472 total_nvlink_bytes=1820852224 weight_to_token_ratio=0.043256 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.202128292e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117252 payload_rows_remote=100988 dup_rows=13820 dup_rows_remote=13800 dedup_ratio=0.105438 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 1702, 0, 0, 0, 0, 7517, 0, 2423, 0, 0, 0, 0, 0, 0, 533, 0, 0, 0, 0, 0, 4511, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 9, 0, 8130, 0, 0, 0, 8188, 1, 0, 0, 991, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 6908, 2, 48, 0, 8178, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16382, 16380, 3404, 16374, 16382, 16264, 15684, 16382] dup_rows_per_dst=[6, 2, 0, 4572, 9024, 20, 114, 82] combine_rows_per_src=[16382, 16380, 3404, 16374, 16382, 16264, 15684, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=889864192 dispatch_meta_bytes=434504 combine_payload_bytes=889864192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=890298696 ingress_bytes=965361664 link_bytes=965361664 token_bytes=1780162888 weight_movement_bytes=75497472 total_nvlink_bytes=1855660360 weight_to_token_ratio=0.042410 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=39808 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.224214475e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119628 payload_rows_remote=103244 dup_rows=11444 dup_rows_remote=11444 dedup_ratio=0.087311 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 7459, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 2687, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 1, 0, 6336, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 10268, 14990, 16384, 16160, 16384, 12674] dup_rows_per_dst=[0, 0, 0, 5944, 5394, 106, 0, 0] combine_rows_per_src=[16384, 16384, 10268, 14990, 16384, 16160, 16384, 12674] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=112556 payload_rows_remote=96172 dup_rows=18516 dup_rows_remote=18516 dedup_ratio=0.141266 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8170, 0, 0, 0, 0, 6638, 0, 80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1832, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 7866, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 13434, 3664, 16362, 16384, 13560] dup_rows_per_dst=[0, 2, 16340, 2, 0, 0, 2172, 0] combine_rows_per_src=[16384, 16384, 16384, 13434, 3664, 16362, 16384, 13560] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=789905408 dispatch_meta_bytes=407392 combine_payload_bytes=789905408 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=790312800 ingress_bytes=1016397824 link_bytes=1016397824 token_bytes=1580218208 weight_movement_bytes=226492416 total_nvlink_bytes=1806710624 weight_to_token_ratio=0.143330 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=57984 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.288935510e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=798949376 dispatch_meta_bytes=436928 combine_payload_bytes=798949376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=799386304 ingress_bytes=874446848 link_bytes=874446848 token_bytes=1598335680 weight_movement_bytes=75497472 total_nvlink_bytes=1673833152 weight_to_token_ratio=0.047235 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=55680 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.108921691e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129954 payload_rows_remote=113570 dup_rows=1118 dup_rows_remote=1118 dedup_ratio=0.008530 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7947, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4879, 0, 0, 0, 0, 3239, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 8192, 0] payload_rows_per_dst=[15894, 16384, 16384, 15756, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 480, 0, 10, 0, 628] combine_rows_per_src=[15894, 16384, 16384, 15756, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=827293696 dispatch_meta_bytes=459152 combine_payload_bytes=827293696 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=827752848 ingress_bytes=1053786112 link_bytes=1053786112 token_bytes=1655046544 weight_movement_bytes=226492416 total_nvlink_bytes=1881538960 weight_to_token_ratio=0.136850 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.336349122e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=845774848 dispatch_meta_bytes=458752 combine_payload_bytes=845774848 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=846233600 ingress_bytes=996769792 link_bytes=996769792 token_bytes=1692008448 weight_movement_bytes=150994944 total_nvlink_bytes=1843003392 weight_to_token_ratio=0.089240 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=62592 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.264044402e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=787841024 dispatch_meta_bytes=458752 combine_payload_bytes=787841024 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=788299776 ingress_bytes=863338496 link_bytes=863338496 token_bytes=1576140800 weight_movement_bytes=75497472 total_nvlink_bytes=1651638272 weight_to_token_ratio=0.047900 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.094834737e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=930365440 dispatch_meta_bytes=458752 combine_payload_bytes=930365440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=930824192 ingress_bytes=1005862912 link_bytes=1005862912 token_bytes=1861189632 weight_movement_bytes=75497472 total_nvlink_bytes=1936687104 weight_to_token_ratio=0.040564 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.275575758e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114768 payload_rows_remote=98384 dup_rows=16304 dup_rows_remote=22 dedup_ratio=0.124390 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8141, 0, 0, 0, 0, 6607, 0, 92, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2105, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 7692, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 13378, 4210, 16260, 16384, 15384] dup_rows_per_dst=[0, 0, 16282, 20, 0, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16384, 13378, 4210, 16260, 16384, 15384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128796 payload_rows_remote=112412 dup_rows=2276 dup_rows_remote=2276 dedup_ratio=0.017365 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8184, 0, 18, 0, 0, 7193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7022, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2277, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8071, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 14388, 16384, 16362, 16384, 16368, 16142, 16384] dup_rows_per_dst=[6, 34, 0, 2236, 0, 0, 0, 0] combine_rows_per_src=[16384, 14388, 16384, 16362, 16384, 16368, 16142, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114414 payload_rows_remote=102148 dup_rows=16658 dup_rows_remote=16658 dedup_ratio=0.127090 router_tokens_per_expert=[0, 0, 0, 0, 9, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 3, 0, 0, 8190, 0, 8131, 1, 2, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 8189, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 6132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 153, 0, 0, 2, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 1, 0, 0, 1930, 0, 2, 0, 0, 0, 8187, 0, 0, 0, 2, 2, 0, 0, 11, 0, 8168, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 12266, 16384, 16384, 3870, 16362] dup_rows_per_dst=[32, 16256, 0, 0, 314, 48, 4, 4] combine_rows_per_src=[16380, 16384, 16384, 12266, 16384, 16384, 3870, 16362] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113456 payload_rows_remote=97072 dup_rows=17616 dup_rows_remote=16380 dedup_ratio=0.134399 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 17, 0, 0, 8021, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7919, 0, 0, 0, 0, 0, 0, 0, 0, 0, 573, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8044, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16338, 34, 16384, 16374, 15174, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 20, 610, 4, 0, 15746, 1236] combine_rows_per_src=[16338, 34, 16384, 16374, 15174, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114680 payload_rows_remote=98298 dup_rows=16392 dup_rows_remote=16348 dedup_ratio=0.125061 router_tokens_per_expert=[0, 0, 0, 0, 6, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 8190, 0, 7957, 0, 1, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 2, 0, 1, 0, 9, 6882, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 209, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 21, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 2, 0, 0, 0, 0, 1312, 0, 28, 0, 0, 0, 8186, 0, 0, 0, 2, 2, 0, 0, 1, 0, 8164, 0, 0] payload_rows_per_dst=[16380, 16378, 16364, 13784, 16384, 16382, 2674, 16334] dup_rows_per_dst=[10, 15908, 0, 2, 410, 44, 14, 4] combine_rows_per_src=[16380, 16378, 16364, 13784, 16384, 16382, 2674, 16334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=805961728 dispatch_meta_bytes=393624 combine_payload_bytes=805961728 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=806355352 ingress_bytes=1183449088 link_bytes=1183449088 token_bytes=1612317080 weight_movement_bytes=377487360 total_nvlink_bytes=1989804440 weight_to_token_ratio=0.234127 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=39808 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.500780027e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=920879104 dispatch_meta_bytes=458752 combine_payload_bytes=920879104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=921337856 ingress_bytes=996376576 link_bytes=996376576 token_bytes=1842216960 weight_movement_bytes=75497472 total_nvlink_bytes=1917714432 weight_to_token_ratio=0.040982 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.263545749e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117984 payload_rows_remote=101636 dup_rows=13088 dup_rows_remote=396 dedup_ratio=0.099854 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8184, 0, 26, 0, 0, 8108, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7913, 0, 0, 0, 0, 0, 0, 0, 0, 0, 442, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8091, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 3370, 16384, 16362, 16384, 16368, 16348, 16384] dup_rows_per_dst=[6, 40, 0, 348, 2, 0, 12692, 0] combine_rows_per_src=[16384, 3370, 16384, 16362, 16384, 16368, 16348, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114674 payload_rows_remote=98290 dup_rows=16398 dup_rows_remote=16398 dedup_ratio=0.125107 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 6800, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3213, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 6370, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 13602, 6426, 16370, 16384, 12740] dup_rows_per_dst=[0, 0, 16384, 2, 10, 0, 2, 0] combine_rows_per_src=[16384, 16384, 16384, 13602, 6426, 16370, 16384, 12740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127286 payload_rows_remote=110902 dup_rows=3786 dup_rows_remote=3786 dedup_ratio=0.028885 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 6, 0, 0, 7803, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6897, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2530, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 37, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7303, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 15606, 16384, 16382, 16384, 16328, 14606, 15212] dup_rows_per_dst=[0, 502, 0, 3210, 74, 0, 0, 0] combine_rows_per_src=[16384, 15606, 16384, 16382, 16384, 16328, 14606, 15212] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=836796416 dispatch_meta_bytes=475224 combine_payload_bytes=836796416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=837271640 ingress_bytes=912293888 link_bytes=912293888 token_bytes=1674068056 weight_movement_bytes=75497472 total_nvlink_bytes=1749565528 weight_to_token_ratio=0.045098 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=54400 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.156917065e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=795213824 dispatch_meta_bytes=453808 combine_payload_bytes=795213824 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=795667632 ingress_bytes=1097203712 link_bytes=1097203712 token_bytes=1590881456 weight_movement_bytes=301989888 total_nvlink_bytes=1892871344 weight_to_token_ratio=0.189826 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65024 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.391408750e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=805257216 dispatch_meta_bytes=458584 combine_payload_bytes=805257216 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=805715800 ingress_bytes=880754688 link_bytes=880754688 token_bytes=1610973016 weight_movement_bytes=75497472 total_nvlink_bytes=1686470488 weight_to_token_ratio=0.046865 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.116920920e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=832602112 dispatch_meta_bytes=408128 combine_payload_bytes=832602112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=833010240 ingress_bytes=908099584 link_bytes=908099584 token_bytes=1665612352 weight_movement_bytes=75497472 total_nvlink_bytes=1741109824 weight_to_token_ratio=0.045327 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=51072 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.151598097e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=805191680 dispatch_meta_bytes=458752 combine_payload_bytes=805191680 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=805650432 ingress_bytes=956186624 link_bytes=956186624 token_bytes=1610842112 weight_movement_bytes=150994944 total_nvlink_bytes=1761837056 weight_to_token_ratio=0.093737 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212579233e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=908509184 dispatch_meta_bytes=458752 combine_payload_bytes=908509184 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=908967936 ingress_bytes=1059504128 link_bytes=1059504128 token_bytes=1817477120 weight_movement_bytes=150994944 total_nvlink_bytes=1968472064 weight_to_token_ratio=0.083079 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.343600371e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130106 payload_rows_remote=113722 dup_rows=966 dup_rows_remote=966 dedup_ratio=0.007370 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 12, 0, 0, 7950, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3570, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4695, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 203, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8146, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 15902, 16384, 15992, 16384, 16384, 16292, 16384] dup_rows_per_dst=[0, 22, 0, 538, 406, 0, 0, 0] combine_rows_per_src=[16384, 15902, 16384, 15992, 16384, 16384, 16292, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114688 dup_rows=56 dup_rows_remote=56 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8192, 0, 0, 15, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16328, 16384, 16384, 16384] dup_rows_per_dst=[0, 32, 0, 0, 0, 0, 24, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16328, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115414 payload_rows_remote=100350 dup_rows=15658 dup_rows_remote=15658 dedup_ratio=0.119461 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1, 0, 0, 8190, 15, 0, 5123, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 7527, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1070, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 8188, 0, 8185, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 30, 0, 0, 0, 2, 2651, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16302, 15064, 2144, 16374, 16384, 16382] dup_rows_per_dst=[8, 10278, 0, 0, 0, 0, 5372, 0] combine_rows_per_src=[16384, 16380, 16302, 15064, 2144, 16374, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=931610624 dispatch_meta_bytes=458752 combine_payload_bytes=931610624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=932069376 ingress_bytes=1007108096 link_bytes=1007108096 token_bytes=1863680000 weight_movement_bytes=75497472 total_nvlink_bytes=1939177472 weight_to_token_ratio=0.040510 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.277154827e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114690 payload_rows_remote=98306 dup_rows=16382 dup_rows_remote=16378 dedup_ratio=0.124985 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 14, 16384, 16384, 16384, 16380, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 16376, 0, 2, 4] combine_rows_per_src=[16376, 14, 16384, 16384, 16384, 16380, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116968 payload_rows_remote=100590 dup_rows=14104 dup_rows_remote=14102 dedup_ratio=0.107605 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 8190, 1, 0, 5303, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 0, 7340, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2006, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 8189, 0, 8187, 8190, 0, 0, 1, 0, 1, 0, 0, 0, 14, 0, 0, 0, 2, 1724, 0, 0, 2, 0, 0, 0, 0, 2, 0, 2, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16380, 16344, 14702, 4014, 16378, 16384, 16382] dup_rows_per_dst=[0, 10614, 0, 0, 0, 2, 3488, 0] combine_rows_per_src=[16384, 16380, 16344, 14702, 4014, 16378, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458976 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939983072 ingress_bytes=1015021568 link_bytes=1015021568 token_bytes=1879507168 weight_movement_bytes=75497472 total_nvlink_bytes=1955004640 weight_to_token_ratio=0.040169 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=52096 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287190223e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120368 payload_rows_remote=103984 dup_rows=10704 dup_rows_remote=10692 dedup_ratio=0.081665 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 8192, 0, 0, 6, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 5704, 16384, 16384, 16360, 16384, 16384, 16384] dup_rows_per_dst=[0, 12, 0, 0, 10680, 0, 12, 0] combine_rows_per_src=[16384, 5704, 16384, 16384, 16360, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130830 payload_rows_remote=114602 dup_rows=242 dup_rows_remote=222 dedup_ratio=0.001846 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 10, 0, 0, 8114, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8030, 0, 0, 0, 0, 0, 0, 0, 0, 0, 273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8150, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16228, 16384, 16384, 16384, 16382, 16300, 16384] dup_rows_per_dst=[0, 20, 0, 222, 0, 0, 0, 0] combine_rows_per_src=[16384, 16228, 16384, 16384, 16384, 16382, 16300, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=6 export_slots=6 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=822067200 dispatch_meta_bytes=464032 combine_payload_bytes=822067200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=822531232 ingress_bytes=897564672 link_bytes=897564672 token_bytes=1644598432 weight_movement_bytes=75497472 total_nvlink_bytes=1720095904 weight_to_token_ratio=0.045906 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.138238346e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121978 payload_rows_remote=105594 dup_rows=9094 dup_rows_remote=9094 dedup_ratio=0.069382 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 96, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8096, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16192, 16384, 7482, 16384] dup_rows_per_dst=[0, 5952, 122, 3002, 0, 18, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16192, 16384, 7482, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=824033280 dispatch_meta_bytes=458768 combine_payload_bytes=824033280 prefetch_bytes=0 reduce_bytes=452984832 egress_bytes=824492048 ingress_bytes=1277018112 link_bytes=1277018112 token_bytes=1648525328 weight_movement_bytes=452984832 total_nvlink_bytes=2101510160 weight_to_token_ratio=0.274782 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=61184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.619438720e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=805322752 dispatch_meta_bytes=458736 combine_payload_bytes=805322752 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=805781488 ingress_bytes=956317696 link_bytes=956317696 token_bytes=1611104240 weight_movement_bytes=150994944 total_nvlink_bytes=1762099184 weight_to_token_ratio=0.093721 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.212745450e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=851836928 dispatch_meta_bytes=458704 combine_payload_bytes=851836928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=852295632 ingress_bytes=927334400 link_bytes=927334400 token_bytes=1704132560 weight_movement_bytes=75497472 total_nvlink_bytes=1779630032 weight_to_token_ratio=0.044303 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=61184 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.175990551e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=938819584 dispatch_meta_bytes=459296 combine_payload_bytes=938819584 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939278880 ingress_bytes=1165312000 link_bytes=1165312000 token_bytes=1878098464 weight_movement_bytes=226492416 total_nvlink_bytes=2104590880 weight_to_token_ratio=0.120597 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=49152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.477779646e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=865026048 dispatch_meta_bytes=458752 combine_payload_bytes=865026048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=865484800 ingress_bytes=940523520 link_bytes=940523520 token_bytes=1730510848 weight_movement_bytes=75497472 total_nvlink_bytes=1806008320 weight_to_token_ratio=0.043627 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.192716212e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131026 payload_rows_remote=114642 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000351 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 21, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16338, 16384, 16384, 16384] dup_rows_per_dst=[0, 42, 0, 0, 0, 0, 4, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16338, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130966 payload_rows_remote=114582 dup_rows=106 dup_rows_remote=94 dedup_ratio=0.000809 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8140, 0, 0, 0, 8192, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16280, 16384, 16384, 16384, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 6, 2, 86, 12, 0, 0, 0] combine_rows_per_src=[16280, 16384, 16384, 16384, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129032 payload_rows_remote=112700 dup_rows=2040 dup_rows_remote=1670 dedup_ratio=0.015564 router_tokens_per_expert=[0, 0, 0, 0, 8189, 0, 0, 12, 0, 0, 2, 0, 8190, 0, 0, 0, 8188, 0, 0, 100, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2, 0, 0, 7198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8006, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 345, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 2, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 1, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 877, 0, 0, 2, 0, 0, 7970, 0, 0, 0, 0, 0, 0, 62, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 14400, 16332, 16384, 16384, 16384, 16380] dup_rows_per_dst=[24, 200, 0, 370, 4, 2, 1314, 126] combine_rows_per_src=[16384, 16384, 14400, 16332, 16384, 16384, 16384, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458752 combine_payload_bytes=939147264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939606016 ingress_bytes=1014644736 link_bytes=1014644736 token_bytes=1878753280 weight_movement_bytes=75497472 total_nvlink_bytes=1954250752 weight_to_token_ratio=0.040185 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286712347e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938655744 dispatch_meta_bytes=458704 combine_payload_bytes=938655744 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=939114448 ingress_bytes=1089650688 link_bytes=1089650688 token_bytes=1877770192 weight_movement_bytes=150994944 total_nvlink_bytes=2028765136 weight_to_token_ratio=0.080412 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381830452e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=129700 payload_rows_remote=113324 dup_rows=1372 dup_rows_remote=1370 dedup_ratio=0.010468 router_tokens_per_expert=[0, 0, 0, 0, 8188, 0, 0, 3, 0, 0, 2, 0, 8190, 0, 0, 0, 8190, 0, 0, 100, 0, 0, 1, 0, 0, 0, 0, 1, 0, 2, 0, 0, 7575, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8091, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 495, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 8190, 0, 1, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 314, 0, 0, 2, 0, 0, 7973, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 15154, 16372, 16384, 16376, 16266, 16380] dup_rows_per_dst=[6, 204, 0, 836, 2, 2, 318, 4] combine_rows_per_src=[16384, 16384, 15154, 16372, 16384, 16376, 16266, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130732 payload_rows_remote=114348 dup_rows=340 dup_rows_remote=340 dedup_ratio=0.002594 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16380, 16052, 16384, 16384, 16380, 16384, 16384, 16384] dup_rows_per_dst=[0, 4, 0, 0, 0, 4, 332, 0] combine_rows_per_src=[16380, 16052, 16384, 16384, 16380, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130856 payload_rows_remote=114476 dup_rows=216 dup_rows_remote=216 dedup_ratio=0.001648 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 10, 0, 0, 0, 0, 8076, 0, 0, 0, 8192, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 0, 0, 42, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16172, 16384, 16384, 16384, 16384, 16384, 16380, 16384] dup_rows_per_dst=[0, 16, 4, 112, 84, 0, 0, 0] combine_rows_per_src=[16172, 16384, 16384, 16384, 16384, 16384, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130944 payload_rows_remote=114560 dup_rows=128 dup_rows_remote=0 dedup_ratio=0.000977 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 64, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16256, 16384, 16384, 16384] dup_rows_per_dst=[0, 128, 0, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16256, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=923238400 dispatch_meta_bytes=457480 combine_payload_bytes=923238400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=923695880 ingress_bytes=998735872 link_bytes=998735872 token_bytes=1846934280 weight_movement_bytes=75497472 total_nvlink_bytes=1922431752 weight_to_token_ratio=0.040877 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.266537668e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127472 payload_rows_remote=111088 dup_rows=3600 dup_rows_remote=3208 dedup_ratio=0.027466 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16242, 16384, 15028, 16286, 16384, 14532, 16232] dup_rows_per_dst=[392, 0, 3208, 0, 0, 0, 0, 0] combine_rows_per_src=[16384, 16242, 16384, 15028, 16286, 16384, 14532, 16232] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21768788456 prefetch_bytes=679477248 reduce_bytes=754974720 weight_movement_bytes=1434451968 egress_bytes=10887141864 ingress_bytes=12316098560 link_bytes=12316557312 total_nvlink_bytes=23203240424 weight_to_token_ratio=0.065895 prefetch_share_of_ingress=0.055170 dedup_ratio=0.040346 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=10 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051930 gemm_inner_pad_upper_bound=9144 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.301594030e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=928350208 dispatch_meta_bytes=458776 combine_payload_bytes=928350208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=928808984 ingress_bytes=1003847680 link_bytes=1003847680 token_bytes=1857159192 weight_movement_bytes=75497472 total_nvlink_bytes=1932656664 weight_to_token_ratio=0.040652 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.273020160e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=936738816 dispatch_meta_bytes=458752 combine_payload_bytes=936738816 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937197568 ingress_bytes=1012236288 link_bytes=1012236288 token_bytes=1873936384 weight_movement_bytes=75497472 total_nvlink_bytes=1949433856 weight_to_token_ratio=0.040288 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283658096e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=937787392 dispatch_meta_bytes=458768 combine_payload_bytes=937787392 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=938246160 ingress_bytes=1088782336 link_bytes=1088782336 token_bytes=1876033552 weight_movement_bytes=150994944 total_nvlink_bytes=2027028496 weight_to_token_ratio=0.080486 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=64256 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.380729260e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20404683176 prefetch_bytes=830472192 reduce_bytes=301989888 weight_movement_bytes=1132462080 egress_bytes=10205102504 ingress_bytes=11332042752 link_bytes=11332042752 total_nvlink_bytes=21537145256 weight_to_token_ratio=0.055500 prefetch_share_of_ingress=0.073285 dedup_ratio=0.101360 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=4 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050551 gemm_inner_pad_upper_bound=10541 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.197552110e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=938475520 dispatch_meta_bytes=458240 combine_payload_bytes=938475520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=938933760 ingress_bytes=1164967936 link_bytes=1164967936 token_bytes=1877409280 weight_movement_bytes=226492416 total_nvlink_bytes=2103901696 weight_to_token_ratio=0.120641 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=52096 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.477343324e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131026 payload_rows_remote=114642 dup_rows=46 dup_rows_remote=46 dedup_ratio=0.000351 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 8192, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16338, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 32, 0, 10, 0, 0, 4, 0] combine_rows_per_src=[16338, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=910032896 dispatch_meta_bytes=457184 combine_payload_bytes=910032896 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=910490080 ingress_bytes=1061027840 link_bytes=1061027840 token_bytes=1820522976 weight_movement_bytes=150994944 total_nvlink_bytes=1971517920 weight_to_token_ratio=0.082940 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.345532652e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20237923368 prefetch_bytes=981467136 reduce_bytes=603979776 weight_movement_bytes=1585446912 egress_bytes=10121720872 ingress_bytes=11701649408 link_bytes=11701649408 total_nvlink_bytes=21823370280 weight_to_token_ratio=0.078340 prefetch_share_of_ingress=0.083874 dedup_ratio=0.095070 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=8 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050858 gemm_inner_pad_upper_bound=10033 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.236611548e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=18967927840 prefetch_bytes=981467136 reduce_bytes=1358954496 weight_movement_bytes=2340421632 egress_bytes=9486556192 ingress_bytes=11821793280 link_bytes=11821793280 total_nvlink_bytes=21308349472 weight_to_token_ratio=0.123388 prefetch_share_of_ingress=0.083022 dedup_ratio=0.139908 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=18 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050858 gemm_inner_pad_upper_bound=10541 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.249308160e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20290288992 prefetch_bytes=1056964608 reduce_bytes=377487360 weight_movement_bytes=1434451968 egress_bytes=10147855712 ingress_bytes=11576885248 link_bytes=11576885248 total_nvlink_bytes=21724740960 weight_to_token_ratio=0.070696 prefetch_share_of_ingress=0.091300 dedup_ratio=0.089238 prefetch_slots=28 unused_prefetch_slots=356 slot_utilization=0.072917 reduce_slots=5 remote_demand=14 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051164 gemm_inner_pad_upper_bound=9779 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.223426672e-04 prefetch_bytes_per_gemm_flop=1.116983252e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19589518232 prefetch_bytes=603979776 reduce_bytes=1434451968 weight_movement_bytes=2038431744 egress_bytes=9797440408 ingress_bytes=11830509568 link_bytes=11830509568 total_nvlink_bytes=21627949976 weight_to_token_ratio=0.104057 prefetch_share_of_ingress=0.051053 dedup_ratio=0.115098 prefetch_slots=16 unused_prefetch_slots=368 slot_utilization=0.041667 reduce_slots=19 remote_demand=8 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050858 gemm_inner_pad_upper_bound=9779 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.250229284e-04 prefetch_bytes_per_gemm_flop=6.382761438e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939147264 dispatch_meta_bytes=458752 combine_payload_bytes=939147264 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=939606016 ingress_bytes=1090142208 link_bytes=1090142208 token_bytes=1878753280 weight_movement_bytes=150994944 total_nvlink_bytes=2029748224 weight_to_token_ratio=0.080370 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=64000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.382453769e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131016 payload_rows_remote=114632 dup_rows=56 dup_rows_remote=44 dedup_ratio=0.000427 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8164, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16328, 16384, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 12, 0, 6, 0, 0, 8, 30] combine_rows_per_src=[16328, 16384, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20853899416 prefetch_bytes=830472192 reduce_bytes=1207959552 weight_movement_bytes=2038431744 egress_bytes=10429612184 ingress_bytes=12462718976 link_bytes=12462718976 total_nvlink_bytes=22892331160 weight_to_token_ratio=0.097748 prefetch_share_of_ingress=0.066637 dedup_ratio=0.067261 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=16 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051471 gemm_inner_pad_upper_bound=9779 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.317040160e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=939065344 dispatch_meta_bytes=458704 combine_payload_bytes=939065344 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=939524048 ingress_bytes=1165557760 link_bytes=1165557760 token_bytes=1878589392 weight_movement_bytes=226492416 total_nvlink_bytes=2105081808 weight_to_token_ratio=0.120565 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.478091304e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20790310320 prefetch_bytes=981467136 reduce_bytes=905969664 weight_movement_bytes=1887436800 egress_bytes=10397873584 ingress_bytes=12279873536 link_bytes=12279873536 total_nvlink_bytes=22677747120 weight_to_token_ratio=0.090784 prefetch_share_of_ingress=0.079925 dedup_ratio=0.072519 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=12 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.049632 gemm_inner_pad_upper_bound=11684 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.297717347e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none + [2026-08-29 23:35:13] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 371.0 | throughput per GPU (TFLOP/s/GPU): 487.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.323159E+00 | loss scale: 1.0 | grad norm: 4.718 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=89834 payload_rows_remote=73450 dup_rows=41238 dup_rows_remote=31878 dedup_ratio=0.314621 router_tokens_per_expert=[0, 0, 0, 1, 0, 701, 0, 0, 0, 3978, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1606, 0, 0, 1392, 3687, 0, 0, 8192, 87, 1420, 1, 0, 0, 0, 0, 0, 2, 49, 0, 2, 1, 2, 0, 2078, 0, 0, 72, 5112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3239, 0, 0, 0, 0, 8192, 0, 0, 0, 1547, 0, 0, 1, 2403, 0, 2312, 5403, 0, 0, 1630, 0, 0, 3, 0, 0, 1, 0, 0, 1, 0, 1421, 0, 0, 0, 0, 1181, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 2765, 377, 0, 11, 0, 0, 0, 8, 0, 0, 0, 0, 6497, 0, 0, 0, 0, 156, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 11680, 6478, 14590, 5034, 6194, 13090] dup_rows_per_dst=[9360, 19832, 2956, 0, 8564, 180, 114, 232] combine_rows_per_src=[16384, 16384, 11680, 6478, 14590, 5034, 6194, 13090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=96180 payload_rows_remote=79798 dup_rows=34892 dup_rows_remote=22806 dedup_ratio=0.266205 router_tokens_per_expert=[2, 0, 0, 1, 0, 640, 0, 0, 0, 3982, 0, 4, 0, 0, 0, 6, 0, 22, 0, 2, 1627, 0, 1, 1330, 1823, 1, 2, 8191, 96, 2080, 1, 0, 0, 2, 0, 0, 2, 2, 0, 17, 0, 0, 0, 2301, 0, 0, 27, 5680, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 3, 3941, 2, 0, 0, 0, 8185, 0, 0, 0, 1516, 2, 0, 2, 1748, 0, 2350, 5557, 0, 0, 1480, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 2195, 2, 0, 2, 0, 1666, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2470, 486, 0, 19, 0, 0, 0, 1, 0, 0, 0, 0, 5733, 0, 0, 0, 0, 307, 1, 0, 1, 0] payload_rows_per_dst=[8748, 8410, 13688, 7892, 11012, 16380, 13668, 16382] dup_rows_per_dst=[522, 1914, 6020, 8, 3184, 7762, 3396, 12086] combine_rows_per_src=[8748, 8410, 13688, 7892, 11012, 16380, 13668, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97854 payload_rows_remote=86326 dup_rows=33218 dup_rows_remote=30570 dedup_ratio=0.253433 router_tokens_per_expert=[1, 0, 3, 0, 0, 503, 0, 0, 0, 4584, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1930, 3, 1, 1279, 1659, 0, 0, 8186, 205, 1346, 0, 0, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 1797, 3, 0, 26, 5256, 0, 0, 0, 0, 0, 1, 0, 4, 0, 0, 0, 3801, 0, 0, 0, 0, 8186, 0, 1, 0, 1951, 0, 2, 0, 1602, 1, 2565, 6178, 0, 0, 1645, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 2326, 1, 0, 0, 0, 1767, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 3453, 451, 1, 0, 0, 0, 1, 2, 5, 0, 1, 2, 4573, 0, 0, 0, 0, 218, 0, 0, 0, 3] payload_rows_per_dst=[16380, 9714, 11528, 16380, 11676, 7658, 14920, 9598] dup_rows_per_dst=[10176, 3132, 2648, 7604, 3858, 532, 5256, 12] combine_rows_per_src=[16380, 9714, 11528, 16380, 11676, 7658, 14920, 9598] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101354 payload_rows_remote=84970 dup_rows=29718 dup_rows_remote=21422 dedup_ratio=0.226730 router_tokens_per_expert=[0, 0, 0, 0, 0, 480, 0, 0, 0, 4451, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1535, 0, 0, 1498, 2869, 0, 0, 8192, 123, 1676, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 1402, 0, 0, 114, 5323, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4148, 0, 0, 0, 0, 8192, 0, 0, 0, 1080, 0, 0, 0, 1735, 0, 1976, 5199, 0, 0, 1257, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2486, 0, 0, 0, 0, 1348, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3453, 537, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6134, 0, 0, 0, 0, 324, 0, 0, 0, 0] payload_rows_per_dst=[11150, 11616, 11508, 16384, 10816, 13480, 13808, 12592] dup_rows_per_dst=[1730, 3788, 2174, 8296, 4050, 4784, 4572, 324] combine_rows_per_src=[11150, 11616, 11508, 16384, 10816, 13480, 13808, 12592] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97438 payload_rows_remote=83524 dup_rows=33634 dup_rows_remote=29292 dedup_ratio=0.256607 router_tokens_per_expert=[0, 1, 0, 1, 0, 747, 0, 0, 6, 4782, 0, 0, 0, 0, 0, 3, 0, 3, 0, 0, 1675, 0, 0, 1723, 2288, 0, 0, 8191, 236, 1692, 0, 2, 0, 0, 1, 1, 0, 26, 0, 1, 4, 0, 0, 1738, 0, 0, 44, 5464, 0, 0, 6, 0, 0, 0, 1, 0, 0, 0, 1, 4456, 0, 0, 0, 0, 8180, 0, 0, 0, 1513, 0, 0, 0, 1725, 0, 1614, 5290, 1, 0, 1467, 2, 0, 0, 3, 3, 0, 1, 0, 0, 0, 2102, 0, 0, 0, 0, 1869, 0, 0, 0, 1, 0, 0, 0, 0, 1, 2, 0, 3351, 482, 1, 0, 0, 0, 0, 7, 0, 4, 0, 0, 4511, 0, 2, 0, 0, 309, 0, 0, 0, 2] payload_rows_per_dst=[10242, 8440, 13100, 8922, 10058, 16378, 13914, 16384] dup_rows_per_dst=[838, 2222, 6034, 6, 2586, 7938, 4342, 9668] combine_rows_per_src=[10242, 8440, 13100, 8922, 10058, 16378, 13914, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99902 payload_rows_remote=83520 dup_rows=31170 dup_rows_remote=24870 dedup_ratio=0.237808 router_tokens_per_expert=[1, 0, 0, 0, 0, 718, 0, 0, 0, 4844, 0, 0, 0, 0, 0, 11, 2, 10, 0, 0, 1933, 0, 0, 1909, 2191, 0, 0, 8191, 151, 1654, 11, 0, 0, 0, 0, 0, 0, 13, 0, 1, 0, 0, 0, 1731, 0, 0, 23, 5477, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3976, 0, 0, 1, 0, 8191, 0, 0, 0, 1623, 0, 0, 0, 1764, 0, 1961, 5930, 0, 0, 1629, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1843, 0, 0, 0, 0, 1306, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3330, 461, 0, 9, 0, 1, 0, 4, 0, 0, 0, 2, 4464, 0, 0, 0, 0, 169, 0, 0, 0, 0] payload_rows_per_dst=[10356, 9820, 12666, 12614, 10932, 16382, 14710, 12422] dup_rows_per_dst=[792, 3356, 4370, 4426, 3022, 6300, 4752, 4152] combine_rows_per_src=[10356, 9820, 12666, 12614, 10932, 16382, 14710, 12422] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=601702400 dispatch_meta_bytes=421312 combine_payload_bytes=601702400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=602123712 ingress_bytes=677199872 link_bytes=677199872 token_bytes=1203826112 weight_movement_bytes=75497472 total_nvlink_bytes=1279323584 weight_to_token_ratio=0.062715 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.044118 gemm_active_groups=12 gemm_peak_group_rows=34688 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.587847608e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=653705216 dispatch_meta_bytes=410416 combine_payload_bytes=653705216 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=654115632 ingress_bytes=729202688 link_bytes=729202688 token_bytes=1307820848 weight_movement_bytes=75497472 total_nvlink_bytes=1383318320 weight_to_token_ratio=0.057728 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.036765 gemm_active_groups=15 gemm_peak_group_rows=43520 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.247316514e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=707182592 dispatch_meta_bytes=467584 combine_payload_bytes=707182592 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=707650176 ingress_bytes=782680064 link_bytes=782680064 token_bytes=1414832768 weight_movement_bytes=75497472 total_nvlink_bytes=1490330240 weight_to_token_ratio=0.053361 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.040441 gemm_active_groups=14 gemm_peak_group_rows=43904 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.925484917e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=696074240 dispatch_meta_bytes=425568 combine_payload_bytes=696074240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=696499808 ingress_bytes=771571712 link_bytes=771571712 token_bytes=1392574048 weight_movement_bytes=75497472 total_nvlink_bytes=1468071520 weight_to_token_ratio=0.054214 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=34944 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.784615377e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=684228608 dispatch_meta_bytes=451264 combine_payload_bytes=684228608 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=684679872 ingress_bytes=759726080 link_bytes=759726080 token_bytes=1368908480 weight_movement_bytes=75497472 total_nvlink_bytes=1444405952 weight_to_token_ratio=0.055152 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=1905 gemm_tail_pad_fraction=0.036765 gemm_active_groups=15 gemm_peak_group_rows=35968 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.634396090e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=684195840 dispatch_meta_bytes=433560 combine_payload_bytes=684195840 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=684629400 ingress_bytes=759693312 link_bytes=759693312 token_bytes=1368825240 weight_movement_bytes=75497472 total_nvlink_bytes=1444322712 weight_to_token_ratio=0.055155 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=1524 gemm_tail_pad_fraction=0.040441 gemm_active_groups=12 gemm_peak_group_rows=38144 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.633980545e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126110 payload_rows_remote=109878 dup_rows=4962 dup_rows_remote=362 dedup_ratio=0.037857 router_tokens_per_expert=[10, 0, 0, 0, 0, 0, 8192, 0, 6111, 0, 0, 27, 0, 0, 2968, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7344, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 136, 0, 8005, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8126, 12, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16232, 14886, 16204, 16256, 14688, 15672, 16020, 16152] dup_rows_per_dst=[4600, 2, 0, 124, 0, 232, 4, 0] combine_rows_per_src=[16232, 14886, 16204, 16256, 14688, 15672, 16020, 16152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116970 payload_rows_remote=100690 dup_rows=14102 dup_rows_remote=14018 dedup_ratio=0.107590 router_tokens_per_expert=[52, 0, 7, 0, 0, 0, 8172, 0, 6854, 0, 0, 203, 0, 0, 1184, 0, 0, 0, 0, 0, 1, 8191, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8136, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 2, 0, 4, 0, 0, 60, 0, 8074, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8121, 54, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[2730, 16382, 16362, 16380, 16296, 16384, 16156, 16280] dup_rows_per_dst=[162, 36, 6, 2, 6992, 134, 6686, 84] combine_rows_per_src=[2730, 16382, 16362, 16380, 16296, 16384, 16156, 16280] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127476 payload_rows_remote=111104 dup_rows=3596 dup_rows_remote=3588 dedup_ratio=0.027435 router_tokens_per_expert=[264, 0, 0, 5, 0, 0, 8185, 6, 6549, 3, 0, 175, 0, 0, 2872, 0, 0, 0, 0, 0, 0, 8185, 6, 0, 0, 25, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 8187, 0, 0, 1, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 1, 7562, 0, 0, 0, 0, 0, 1, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 7558, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 7678, 49, 5, 0, 0, 0, 0, 0, 0, 1] payload_rows_per_dst=[16322, 16382, 16372, 16384, 15136, 16374, 15126, 15380] dup_rows_per_dst=[3426, 56, 8, 0, 0, 10, 0, 96] combine_rows_per_src=[16322, 16382, 16372, 16384, 15136, 16374, 15126, 15380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125508 payload_rows_remote=110340 dup_rows=5564 dup_rows_remote=5560 dedup_ratio=0.042450 router_tokens_per_expert=[1033, 0, 0, 0, 0, 0, 8173, 0, 4279, 0, 0, 91, 0, 0, 3195, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7515, 0, 0, 0, 0, 0, 2, 0, 0, 1, 0, 0, 0, 0, 0, 5, 0, 13, 0, 0, 40, 0, 7576, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7637, 1162, 0, 0, 0, 0, 0, 0, 3, 0] payload_rows_per_dst=[13556, 16384, 16370, 16380, 15108, 16384, 15168, 16158] dup_rows_per_dst=[1678, 382, 40, 10, 1884, 120, 4, 1446] combine_rows_per_src=[13556, 16384, 16370, 16380, 15108, 16384, 15168, 16158] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130194 payload_rows_remote=113810 dup_rows=878 dup_rows_remote=878 dedup_ratio=0.006699 router_tokens_per_expert=[173, 0, 0, 0, 0, 0, 8191, 0, 6441, 0, 0, 23, 0, 0, 1853, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8013, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 3, 0, 0, 1, 0, 8076, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7935, 189, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16326, 16384, 16384, 16384, 16026, 16384, 16152, 16154] dup_rows_per_dst=[654, 66, 54, 0, 0, 10, 0, 94] combine_rows_per_src=[16326, 16384, 16384, 16384, 16026, 16384, 16152, 16154] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=900120576 dispatch_meta_bytes=440960 combine_payload_bytes=900120576 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=900561536 ingress_bytes=1202110464 link_bytes=1202110464 token_bytes=1800682112 weight_movement_bytes=301989888 total_nvlink_bytes=2102672000 weight_to_token_ratio=0.167709 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1397 gemm_tail_pad_fraction=0.047794 gemm_active_groups=11 gemm_peak_group_rows=40064 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.524445278e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=824852480 dispatch_meta_bytes=458832 combine_payload_bytes=824852480 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=825311312 ingress_bytes=900349952 link_bytes=900349952 token_bytes=1650163792 weight_movement_bytes=75497472 total_nvlink_bytes=1725661264 weight_to_token_ratio=0.045752 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=63616 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.141770473e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126138 payload_rows_remote=109754 dup_rows=4934 dup_rows_remote=4880 dedup_ratio=0.037643 router_tokens_per_expert=[402, 0, 0, 0, 0, 0, 8182, 0, 6153, 0, 0, 193, 0, 0, 2856, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 431, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 129, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7201, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 0, 22, 0, 0, 1, 0, 7098, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7901, 382, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16068, 16384, 16380, 16384, 14402, 16384, 14196, 15940] dup_rows_per_dst=[3140, 862, 252, 0, 0, 54, 0, 626] combine_rows_per_src=[16068, 16384, 16380, 16384, 14402, 16384, 14196, 15940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=910163968 dispatch_meta_bytes=458768 combine_payload_bytes=910163968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=910622736 ingress_bytes=985661440 link_bytes=985661440 token_bytes=1820786704 weight_movement_bytes=75497472 total_nvlink_bytes=1896284176 weight_to_token_ratio=0.041464 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.249957448e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=903905280 dispatch_meta_bytes=463600 combine_payload_bytes=903905280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=904368880 ingress_bytes=979402752 link_bytes=979402752 token_bytes=1808274160 weight_movement_bytes=75497472 total_nvlink_bytes=1883771632 weight_to_token_ratio=0.041751 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=62208 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.242020551e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=932331520 dispatch_meta_bytes=458752 combine_payload_bytes=932331520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=932790272 ingress_bytes=1158823936 link_bytes=1158823936 token_bytes=1865121792 weight_movement_bytes=226492416 total_nvlink_bytes=2091614208 weight_to_token_ratio=0.121436 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.469551868e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=899104768 dispatch_meta_bytes=458536 combine_payload_bytes=899104768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=899563304 ingress_bytes=974602240 link_bytes=974602240 token_bytes=1798668072 weight_movement_bytes=75497472 total_nvlink_bytes=1874165544 weight_to_token_ratio=0.041974 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.235932826e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=101868 payload_rows_remote=90322 dup_rows=29204 dup_rows_remote=25654 dedup_ratio=0.222809 router_tokens_per_expert=[0, 0, 0, 0, 1, 456, 0, 0, 0, 4447, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2076, 0, 0, 1548, 1955, 0, 0, 8192, 170, 1551, 16, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 1489, 0, 0, 41, 6032, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4363, 0, 0, 0, 0, 8191, 0, 0, 0, 1603, 0, 1, 0, 1725, 0, 2238, 5624, 0, 0, 1981, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 2157, 0, 0, 0, 0, 1024, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 3065, 441, 0, 7, 0, 0, 0, 0, 0, 0, 0, 1, 4805, 0, 0, 0, 0, 296, 0, 1, 0, 0] payload_rows_per_dst=[9358, 11172, 12796, 16384, 11546, 16382, 14120, 10110] dup_rows_per_dst=[450, 3460, 2398, 8726, 3550, 6366, 4158, 96] combine_rows_per_src=[9358, 11172, 12796, 16384, 11546, 16382, 14120, 10110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=97240 payload_rows_remote=84962 dup_rows=33832 dup_rows_remote=28298 dedup_ratio=0.258118 router_tokens_per_expert=[0, 0, 0, 2, 4, 489, 0, 0, 0, 3606, 0, 0, 0, 0, 0, 4, 0, 13, 0, 0, 961, 0, 0, 1603, 2183, 0, 0, 8192, 142, 1599, 0, 0, 0, 0, 0, 0, 2, 10, 0, 0, 0, 0, 0, 3035, 0, 0, 159, 5547, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 2688, 0, 0, 0, 0, 8188, 0, 0, 0, 1194, 0, 0, 0, 1628, 0, 2663, 5435, 0, 0, 2008, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 971, 0, 0, 0, 0, 1783, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3853, 573, 0, 34, 0, 0, 0, 1, 0, 0, 0, 1, 6738, 0, 0, 0, 0, 222, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12278, 12918, 12644, 13214, 5298, 10976, 13528] dup_rows_per_dst=[8202, 5534, 4588, 4308, 8414, 218, 2172, 396] combine_rows_per_src=[16384, 12278, 12918, 12644, 13214, 5298, 10976, 13528] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121660 payload_rows_remote=105276 dup_rows=9412 dup_rows_remote=9384 dedup_ratio=0.071808 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8168, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 136, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7440, 0, 4007, 0, 0, 0, 1, 0, 0, 4779, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 5] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16384, 13924, 16250, 9568] dup_rows_per_dst=[28, 52, 2414, 0, 272, 0, 6646, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16384, 13924, 16250, 9568] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=110562 payload_rows_remote=94178 dup_rows=20510 dup_rows_remote=2960 dedup_ratio=0.156479 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 28, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8162, 0, 8192, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5053, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 98, 0, 0, 0, 0, 0, 0, 0, 0, 7628, 0, 2537, 0, 0, 0, 6, 0, 0, 1012, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 39] payload_rows_per_dst=[16222, 16024, 11236, 16192, 10106, 16384, 8014, 16384] dup_rows_per_dst=[26, 46, 162, 6, 0, 200, 2520, 17550] combine_rows_per_src=[16222, 16024, 11236, 16192, 10106, 16384, 8014, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119296 payload_rows_remote=102918 dup_rows=11776 dup_rows_remote=8506 dedup_ratio=0.089844 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 3, 0, 8186, 0, 566, 0, 0, 7469, 1, 1458, 0, 0, 0, 0, 7793, 0, 8186, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 82, 0, 0, 0, 0, 0, 0, 6, 0, 0, 2687, 0, 0, 0, 0, 0, 10, 0, 5, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 368, 0, 0, 0, 0, 0, 0, 0, 0, 7525, 0, 1129, 5, 0, 0, 0, 0, 0, 3615, 0, 0, 2, 8186, 0, 6, 0, 0, 5, 0, 0, 0, 8186, 0, 3] payload_rows_per_dst=[16382, 16382, 16378, 16372, 16384, 14984, 15170, 7244] dup_rows_per_dst=[78, 2, 3270, 164, 5404, 692, 2148, 18] combine_rows_per_src=[16382, 16382, 16378, 16372, 16384, 14984, 15170, 7244] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114442 payload_rows_remote=112786 dup_rows=16630 dup_rows_remote=16630 dedup_ratio=0.126877 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 68, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8006, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7428, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 650, 0, 0, 0, 0, 0, 0, 0, 0, 7572, 0, 816, 0, 0, 0, 12, 0, 0, 7, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8] payload_rows_per_dst=[16384, 16384, 16012, 16384, 14856, 16382, 1656, 16384] dup_rows_per_dst=[20, 136, 0, 0, 0, 1300, 0, 15174] combine_rows_per_src=[16384, 16384, 16012, 16384, 14856, 16382, 1656, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=862420992 dispatch_meta_bytes=458640 combine_payload_bytes=862420992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=862879632 ingress_bytes=937918464 link_bytes=937918464 token_bytes=1725300624 weight_movement_bytes=75497472 total_nvlink_bytes=1800798096 weight_to_token_ratio=0.043759 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.189412634e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=739917824 dispatch_meta_bytes=463904 combine_payload_bytes=739917824 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=740381728 ingress_bytes=1041907712 link_bytes=1041907712 token_bytes=1480299552 weight_movement_bytes=301989888 total_nvlink_bytes=1782289440 weight_to_token_ratio=0.204006 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=1778 gemm_tail_pad_fraction=0.042279 gemm_active_groups=14 gemm_peak_group_rows=16000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.321285638e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118096 payload_rows_remote=101712 dup_rows=12976 dup_rows_remote=12976 dedup_ratio=0.098999 router_tokens_per_expert=[0, 0, 1, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 26, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 8103, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2925, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 207, 0, 0, 0, 0, 0, 0, 0, 0, 8023, 0, 3370, 0, 0, 0, 0, 0, 0, 1859, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 59] payload_rows_per_dst=[16384, 16384, 16206, 16384, 16384, 16382, 16136, 3836] dup_rows_per_dst=[10, 52, 0, 0, 5850, 414, 6650, 0] combine_rows_per_src=[16384, 16384, 16206, 16384, 16384, 16382, 16136, 3836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=843104256 dispatch_meta_bytes=445696 combine_payload_bytes=843104256 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=843549952 ingress_bytes=1145094144 link_bytes=1145094144 token_bytes=1686654208 weight_movement_bytes=301989888 total_nvlink_bytes=1988644096 weight_to_token_ratio=0.179047 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=62080 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.452140559e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=771506176 dispatch_meta_bytes=388552 combine_payload_bytes=771506176 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=771894728 ingress_bytes=997998592 link_bytes=997998592 token_bytes=1543400904 weight_movement_bytes=226492416 total_nvlink_bytes=1769893320 weight_to_token_ratio=0.146749 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.049632 gemm_active_groups=9 gemm_peak_group_rows=27648 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265602694e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126136 payload_rows_remote=109752 dup_rows=4936 dup_rows_remote=3760 dedup_ratio=0.037659 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8147, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7548, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 588, 0, 0, 0, 0, 0, 0, 0, 0, 7544, 0, 519, 0, 0, 0, 1, 0, 0, 52, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 152] payload_rows_per_dst=[16384, 16384, 16294, 16384, 15096, 16384, 12826, 16384] dup_rows_per_dst=[2, 48, 0, 0, 0, 1176, 634, 3076] combine_rows_per_src=[16384, 16384, 16294, 16384, 15096, 16384, 12826, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=696008704 dispatch_meta_bytes=453040 combine_payload_bytes=696008704 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=696461744 ingress_bytes=997998592 link_bytes=997998592 token_bytes=1392470448 weight_movement_bytes=301989888 total_nvlink_bytes=1694460336 weight_to_token_ratio=0.216873 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.038603 gemm_active_groups=17 gemm_peak_group_rows=13440 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265602694e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=923942912 dispatch_meta_bytes=517664 combine_payload_bytes=923942912 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=924460576 ingress_bytes=1074937856 link_bytes=1074937856 token_bytes=1848403488 weight_movement_bytes=150994944 total_nvlink_bytes=1999398432 weight_to_token_ratio=0.081689 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=43136 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.363172510e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=833224704 dispatch_meta_bytes=458752 combine_payload_bytes=833224704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=833683456 ingress_bytes=908722176 link_bytes=908722176 token_bytes=1666908160 weight_movement_bytes=75497472 total_nvlink_bytes=1742405632 weight_to_token_ratio=0.045292 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.152387631e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=899088384 dispatch_meta_bytes=454048 combine_payload_bytes=899088384 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=899542432 ingress_bytes=974585856 link_bytes=974585856 token_bytes=1798630816 weight_movement_bytes=75497472 total_nvlink_bytes=1874128288 weight_to_token_ratio=0.041975 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63616 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.235912049e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127030 payload_rows_remote=110646 dup_rows=4042 dup_rows_remote=3704 dedup_ratio=0.030838 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 0, 8107, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8098, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16030, 16376, 16378, 16384, 12710] dup_rows_per_dst=[338, 8, 350, 0, 0, 3322, 24, 0] combine_rows_per_src=[16384, 16384, 16384, 16030, 16376, 16378, 16384, 12710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128242 payload_rows_remote=112288 dup_rows=2830 dup_rows_remote=2716 dedup_ratio=0.021591 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8053, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 4, 0, 0, 0, 51, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 828, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 1, 0, 6, 0, 0, 0, 0, 8180, 0, 0, 0, 7675, 0, 0, 0, 0, 45, 0, 0, 21, 0, 0, 0, 0, 5, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 508, 0, 0, 0, 7971, 8186, 0, 0, 0, 0, 5, 0, 7588, 8186, 0, 0, 6] payload_rows_per_dst=[16106, 16384, 15954, 16378, 16374, 15484, 16374, 15188] dup_rows_per_dst=[0, 28, 114, 1654, 0, 8, 1016, 10] combine_rows_per_src=[16106, 16384, 15954, 16378, 16374, 15484, 16374, 15188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124648 payload_rows_remote=108578 dup_rows=6424 dup_rows_remote=728 dedup_ratio=0.049011 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 314, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 8180, 0, 0, 0, 8033, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8185, 8192, 0, 0, 0, 0, 0, 0, 8035, 8191, 0, 0, 0] payload_rows_per_dst=[16372, 16248, 13348, 15090, 16364, 16074, 15082, 16070] dup_rows_per_dst=[0, 4, 618, 0, 72, 0, 34, 5696] combine_rows_per_src=[16372, 16248, 13348, 15090, 16364, 16074, 15082, 16070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=906412032 dispatch_meta_bytes=457400 combine_payload_bytes=906412032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=906869432 ingress_bytes=981909504 link_bytes=981909504 token_bytes=1813281464 weight_movement_bytes=75497472 total_nvlink_bytes=1888778936 weight_to_token_ratio=0.041636 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.245199465e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130282 payload_rows_remote=113898 dup_rows=790 dup_rows_remote=642 dedup_ratio=0.006027 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8172, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 43, 0, 0, 0, 0, 0, 0, 0, 0, 288, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 7835, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 8192, 0, 0, 0, 0, 0, 0, 8179, 8192, 0, 0, 0] payload_rows_per_dst=[16364, 16384, 16384, 16384, 16354, 15670, 16384, 16358] dup_rows_per_dst=[0, 86, 556, 0, 0, 0, 148, 0] combine_rows_per_src=[16364, 16384, 16384, 16384, 16354, 15670, 16384, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128046 payload_rows_remote=113500 dup_rows=3026 dup_rows_remote=3026 dedup_ratio=0.023087 router_tokens_per_expert=[4, 0, 0, 0, 0, 0, 8185, 0, 5745, 0, 0, 28, 0, 0, 3815, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7273, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 56, 0, 7646, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 12, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16384, 16370, 16384, 14546, 16384, 15292, 16312] dup_rows_per_dst=[2810, 76, 0, 0, 0, 112, 4, 24] combine_rows_per_src=[16374, 16384, 16370, 16384, 14546, 16384, 15292, 16312] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130690 payload_rows_remote=114306 dup_rows=382 dup_rows_remote=382 dedup_ratio=0.002914 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 181, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8028, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8165, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 16056, 16384, 16330] dup_rows_per_dst=[0, 6, 362, 0, 0, 0, 14, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 16056, 16384, 16330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130320 payload_rows_remote=113936 dup_rows=752 dup_rows_remote=730 dedup_ratio=0.005737 router_tokens_per_expert=[60, 0, 0, 0, 0, 0, 8122, 0, 5716, 0, 0, 8, 0, 0, 2713, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7997, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 10, 0, 8132, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8109, 54, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16358, 16384, 16254, 16384, 15994, 16384, 16264, 16298] dup_rows_per_dst=[636, 22, 16, 0, 0, 50, 0, 28] combine_rows_per_src=[16358, 16384, 16254, 16384, 15994, 16384, 16264, 16298] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=919863296 dispatch_meta_bytes=460016 combine_payload_bytes=919863296 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=920323312 ingress_bytes=995360768 link_bytes=995360768 token_bytes=1840186608 weight_movement_bytes=75497472 total_nvlink_bytes=1915684080 weight_to_token_ratio=0.041027 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.262257561e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=889470976 dispatch_meta_bytes=437224 combine_payload_bytes=889470976 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=889908200 ingress_bytes=1266958336 link_bytes=1266958336 token_bytes=1779379176 weight_movement_bytes=377487360 total_nvlink_bytes=2156866536 weight_to_token_ratio=0.212146 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=62720 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.606681508e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130366 payload_rows_remote=114474 dup_rows=706 dup_rows_remote=706 dedup_ratio=0.005386 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 331, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 7946, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 8192, 0, 0, 0, 0, 0, 0, 8088, 8192, 0, 0, 0] payload_rows_per_dst=[16380, 16384, 16384, 16384, 16382, 15892, 16384, 16176] dup_rows_per_dst=[0, 2, 660, 0, 0, 0, 44, 0] combine_rows_per_src=[16380, 16384, 16384, 16384, 16382, 15892, 16384, 16176] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=933052416 dispatch_meta_bytes=458160 combine_payload_bytes=933052416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=933510576 ingress_bytes=1008549888 link_bytes=1008549888 token_bytes=1866562992 weight_movement_bytes=75497472 total_nvlink_bytes=1942060464 weight_to_token_ratio=0.040447 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.278983222e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=929792000 dispatch_meta_bytes=466104 combine_payload_bytes=929792000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=930258104 ingress_bytes=1005289472 link_bytes=1005289472 token_bytes=1860050104 weight_movement_bytes=75497472 total_nvlink_bytes=1935547576 weight_to_token_ratio=0.040589 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=61056 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.274848556e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=936394752 dispatch_meta_bytes=458752 combine_payload_bytes=936394752 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=936853504 ingress_bytes=1162887168 link_bytes=1162887168 token_bytes=1873248256 weight_movement_bytes=226492416 total_nvlink_bytes=2099740672 weight_to_token_ratio=0.120909 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=64768 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.474704618e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=933363712 dispatch_meta_bytes=458664 combine_payload_bytes=933363712 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=933822376 ingress_bytes=1084358656 link_bytes=1084358656 token_bytes=1867186088 weight_movement_bytes=150994944 total_nvlink_bytes=2018181032 weight_to_token_ratio=0.080868 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=64384 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.375119411e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=937771008 dispatch_meta_bytes=460720 combine_payload_bytes=937771008 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938231728 ingress_bytes=1013268480 link_bytes=1013268480 token_bytes=1876002736 weight_movement_bytes=75497472 total_nvlink_bytes=1951500208 weight_to_token_ratio=0.040244 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=63744 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284967061e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121558 payload_rows_remote=105174 dup_rows=9514 dup_rows_remote=3412 dedup_ratio=0.072586 router_tokens_per_expert=[0, 0, 8190, 0, 0, 8192, 0, 0, 0, 2409, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 4186, 0, 0, 0, 0, 8117, 0, 8150, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1725, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 15144, 8522, 16384, 16132, 16300, 16334, 16358] dup_rows_per_dst=[6102, 0, 6, 2, 3400, 4, 0, 0] combine_rows_per_src=[16384, 15144, 8522, 16384, 16132, 16300, 16334, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=108198 payload_rows_remote=96102 dup_rows=22874 dup_rows_remote=22868 dedup_ratio=0.174515 router_tokens_per_expert=[0, 0, 8186, 0, 0, 8182, 0, 0, 6, 7785, 0, 0, 0, 8186, 2860, 0, 0, 0, 0, 0, 0, 0, 6, 0, 1, 0, 0, 0, 0, 0, 481, 0, 0, 0, 6044, 3, 4, 0, 0, 7789, 0, 7779, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 7336, 0, 0, 0, 0, 0, 0, 0, 0, 354, 0, 0, 0, 0, 0, 0, 0, 0, 0, 432, 16, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 3, 0, 0, 20, 0, 6, 0, 0, 0, 0, 29, 0, 0, 0, 0, 0, 0, 1, 5, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 12096, 15070, 16234, 15580, 16380, 70] dup_rows_per_dst=[21290, 964, 6, 316, 240, 4, 54, 0] combine_rows_per_src=[16384, 16384, 12096, 15070, 16234, 15580, 16380, 70] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111350 payload_rows_remote=94966 dup_rows=19722 dup_rows_remote=3356 dedup_ratio=0.150467 router_tokens_per_expert=[0, 0, 8182, 0, 0, 8192, 0, 0, 0, 1502, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 7816, 0, 0, 0, 0, 7221, 0, 7979, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3004, 16364, 12342, 16384, 14478, 16010, 16384, 16384] dup_rows_per_dst=[0, 4, 0, 16, 68, 3268, 0, 16366] combine_rows_per_src=[3004, 16364, 12342, 16384, 14478, 16010, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=861585408 dispatch_meta_bytes=434344 combine_payload_bytes=861585408 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=862019752 ingress_bytes=1239072768 link_bytes=1239072768 token_bytes=1723605160 weight_movement_bytes=377487360 total_nvlink_bytes=2101092520 weight_to_token_ratio=0.219010 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=41856 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.571318683e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116576 payload_rows_remote=100192 dup_rows=14496 dup_rows_remote=14496 dedup_ratio=0.110596 router_tokens_per_expert=[0, 0, 8180, 0, 0, 8192, 0, 0, 0, 2087, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 21, 0, 0, 0, 7189, 0, 0, 0, 0, 7596, 0, 7708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8156, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[4174, 16360, 16332, 16328, 15196, 15418, 16384, 16384] dup_rows_per_dst=[0, 42, 13010, 1154, 20, 26, 0, 244] combine_rows_per_src=[4174, 16360, 16332, 16328, 15196, 15418, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118784 payload_rows_remote=108190 dup_rows=12288 dup_rows_remote=9712 dedup_ratio=0.093750 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 8190, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 17, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1454, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 34, 0, 0, 0, 0, 0, 0, 0, 0, 7226, 0, 3972, 0, 0, 0, 0, 0, 0, 3574, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 121] payload_rows_per_dst=[16384, 16384, 16380, 16384, 10594, 16360, 16158, 10140] dup_rows_per_dst=[0, 0, 0, 34, 2576, 68, 6238, 3372] combine_rows_per_src=[16384, 16384, 16380, 16384, 10594, 16360, 16158, 10140] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128442 payload_rows_remote=112140 dup_rows=2630 dup_rows_remote=2630 dedup_ratio=0.020065 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1130, 0, 0, 0, 8192, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 7331, 0, 0, 0, 0, 7993, 0, 7989, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8151, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 6, 116, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 14662, 16302, 16018, 15978, 16384, 16330] dup_rows_per_dst=[2264, 146, 0, 0, 220, 0, 0, 0] combine_rows_per_src=[16384, 16384, 14662, 16302, 16018, 15978, 16384, 16330] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=787267584 dispatch_meta_bytes=475880 combine_payload_bytes=787267584 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=787743464 ingress_bytes=1089257472 link_bytes=1089257472 token_bytes=1575011048 weight_movement_bytes=301989888 total_nvlink_bytes=1877000936 weight_to_token_ratio=0.191738 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=50688 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.381331799e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119372 payload_rows_remote=103148 dup_rows=11700 dup_rows_remote=11700 dedup_ratio=0.089264 router_tokens_per_expert=[0, 0, 8191, 0, 0, 8192, 0, 0, 0, 2293, 0, 0, 0, 8191, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50, 0, 0, 0, 6180, 0, 0, 0, 0, 7957, 0, 8112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[5732, 16382, 16016, 16354, 15914, 16224, 16384, 16366] dup_rows_per_dst=[506, 100, 11076, 0, 18, 0, 0, 0] combine_rows_per_src=[5732, 16382, 16016, 16354, 15914, 16224, 16384, 16366] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=777961472 dispatch_meta_bytes=393288 combine_payload_bytes=777961472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=778354760 ingress_bytes=853458944 link_bytes=853458944 token_bytes=1556316232 weight_movement_bytes=75497472 total_nvlink_bytes=1631813704 weight_to_token_ratio=0.048510 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=57344 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082306074e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126466 payload_rows_remote=110082 dup_rows=4606 dup_rows_remote=4580 dedup_ratio=0.035141 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8169, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 391, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 0, 0, 0, 0, 0, 6705, 0, 3308, 0, 0, 0, 0, 0, 0, 5957, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 14] payload_rows_per_dst=[16384, 16384, 16338, 16384, 16384, 16384, 16278, 11930] dup_rows_per_dst=[0, 26, 0, 0, 782, 38, 3748, 12] combine_rows_per_src=[16384, 16384, 16338, 16384, 16384, 16384, 16278, 11930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=820772864 dispatch_meta_bytes=458752 combine_payload_bytes=820772864 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=821231616 ingress_bytes=896270336 link_bytes=896270336 token_bytes=1642004480 weight_movement_bytes=75497472 total_nvlink_bytes=1717501952 weight_to_token_ratio=0.045979 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.136596946e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 4, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=918650880 dispatch_meta_bytes=459080 combine_payload_bytes=918650880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=919109960 ingress_bytes=994148352 link_bytes=994148352 token_bytes=1837760840 weight_movement_bytes=75497472 total_nvlink_bytes=1913258312 weight_to_token_ratio=0.041081 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=64640 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.260720047e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=844988416 dispatch_meta_bytes=459392 combine_payload_bytes=844988416 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=845447808 ingress_bytes=995983360 link_bytes=995983360 token_bytes=1690436224 weight_movement_bytes=150994944 total_nvlink_bytes=1841431168 weight_to_token_ratio=0.089323 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.263047096e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=886292480 dispatch_meta_bytes=471608 combine_payload_bytes=886292480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=886764088 ingress_bytes=961789952 link_bytes=961789952 token_bytes=1773056568 weight_movement_bytes=75497472 total_nvlink_bytes=1848554040 weight_to_token_ratio=0.042580 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=38016 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.219685041e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114038 payload_rows_remote=97654 dup_rows=17034 dup_rows_remote=5384 dedup_ratio=0.129959 router_tokens_per_expert=[0, 0, 8192, 1, 0, 1958, 86, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2632, 0, 8192, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 69, 0, 0, 0, 3, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 3403, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8101, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 149, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 6940, 16356, 8838, 16368] dup_rows_per_dst=[11650, 76, 5268, 4, 10, 0, 26, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 6940, 16356, 8838, 16368] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=901791744 dispatch_meta_bytes=458648 combine_payload_bytes=901791744 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=902250392 ingress_bytes=977289216 link_bytes=977289216 token_bytes=1804042136 weight_movement_bytes=75497472 total_nvlink_bytes=1879539608 weight_to_token_ratio=0.041849 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.239340290e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=109068 payload_rows_remote=103756 dup_rows=22004 dup_rows_remote=22000 dedup_ratio=0.167877 router_tokens_per_expert=[0, 0, 8001, 493, 0, 425, 4425, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 5, 21, 0, 173, 0, 2440, 0, 0, 0, 1784, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6109, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 6, 0, 0, 0, 0, 4237, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 8186, 8145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4643, 0, 0, 0, 8186, 2, 0, 0, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16376, 16302, 5312, 13452, 8486, 16384, 16372, 16384] dup_rows_per_dst=[10324, 2, 4, 2344, 0, 28, 9286, 16] combine_rows_per_src=[16376, 16302, 5312, 13452, 8486, 16384, 16372, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=799981568 dispatch_meta_bytes=412152 combine_payload_bytes=799981568 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=800393720 ingress_bytes=950976512 link_bytes=950976512 token_bytes=1600375288 weight_movement_bytes=150994944 total_nvlink_bytes=1751370232 weight_to_token_ratio=0.094350 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.051471 gemm_active_groups=8 gemm_peak_group_rows=42624 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.205972077e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116460 payload_rows_remote=100110 dup_rows=14612 dup_rows_remote=14512 dedup_ratio=0.111481 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1894, 164, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 311, 0, 8192, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 25, 0, 0, 0, 9, 8174, 0, 0, 0, 0, 2, 0, 0, 0, 5845, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8111, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42, 0, 0, 0, 8175, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[3986, 16376, 14404, 16384, 16384, 16348, 16228, 16350] dup_rows_per_dst=[130, 0, 582, 296, 11654, 108, 1742, 100] combine_rows_per_src=[3986, 16376, 14404, 16384, 16384, 16348, 16228, 16350] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118596 payload_rows_remote=104388 dup_rows=12476 dup_rows_remote=12470 dedup_ratio=0.095184 router_tokens_per_expert=[0, 0, 8192, 2, 0, 3080, 62, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 9, 0, 8192, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 11, 0, 0, 0, 35, 8178, 0, 0, 0, 0, 1, 0, 0, 0, 6051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7104, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6162, 16384, 16384, 16384, 16384, 16356, 14208, 16334] dup_rows_per_dst=[126, 12, 48, 88, 12196, 0, 6, 0] combine_rows_per_src=[6162, 16384, 16384, 16384, 16384, 16356, 14208, 16334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120672 payload_rows_remote=104292 dup_rows=10400 dup_rows_remote=10400 dedup_ratio=0.079346 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1974, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 2, 0, 8192, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 3, 0, 0, 0, 1, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 6324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 8188, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[6830, 16384, 16384, 16384, 15856, 16380, 16078, 16376] dup_rows_per_dst=[774, 0, 18, 44, 9554, 0, 10, 0] combine_rows_per_src=[6830, 16384, 16384, 16384, 15856, 16380, 16078, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 2, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126620 payload_rows_remote=110236 dup_rows=4452 dup_rows_remote=4338 dedup_ratio=0.033966 router_tokens_per_expert=[0, 0, 8192, 4, 0, 1567, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 561, 0, 8191, 0, 0, 0, 57, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 1, 0, 0, 0, 6256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7905, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 12512, 16384, 15810, 16378] dup_rows_per_dst=[3150, 0, 1124, 114, 2, 0, 62, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 12512, 16384, 15810, 16378] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=849969152 dispatch_meta_bytes=503024 combine_payload_bytes=849969152 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=850472176 ingress_bytes=1076461568 link_bytes=1076461568 token_bytes=1700441328 weight_movement_bytes=226492416 total_nvlink_bytes=1926933744 weight_to_token_ratio=0.133196 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=58880 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.365104792e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130556 payload_rows_remote=114172 dup_rows=516 dup_rows_remote=516 dedup_ratio=0.003937 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 243, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8015, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8123, 8192, 0, 0, 0] payload_rows_per_dst=[16360, 16384, 16384, 16384, 16384, 16030, 16384, 16246] dup_rows_per_dst=[0, 2, 486, 0, 0, 0, 28, 0] combine_rows_per_src=[16360, 16384, 16384, 16384, 16384, 16030, 16384, 16246] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=820101120 dispatch_meta_bytes=458488 combine_payload_bytes=820101120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=820559608 ingress_bytes=895598592 link_bytes=895598592 token_bytes=1640660728 weight_movement_bytes=75497472 total_nvlink_bytes=1716158200 weight_to_token_ratio=0.046017 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.135745080e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 4, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=855146496 dispatch_meta_bytes=467432 combine_payload_bytes=855146496 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=855613928 ingress_bytes=1157136384 link_bytes=1157136384 token_bytes=1710760424 weight_movement_bytes=301989888 total_nvlink_bytes=2012750312 weight_to_token_ratio=0.176524 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.056985 gemm_active_groups=4 gemm_peak_group_rows=59776 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.467411814e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130748 payload_rows_remote=114364 dup_rows=324 dup_rows_remote=324 dedup_ratio=0.002472 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 160, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 8079, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 8144, 8192, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16382, 16158, 16384, 16288] dup_rows_per_dst=[0, 0, 320, 0, 0, 0, 4, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16382, 16158, 16384, 16288] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=854360064 dispatch_meta_bytes=458768 combine_payload_bytes=854360064 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=854818832 ingress_bytes=1005355008 link_bytes=1005355008 token_bytes=1709178896 weight_movement_bytes=150994944 total_nvlink_bytes=1860173840 weight_to_token_ratio=0.088344 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.274931665e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=903053312 dispatch_meta_bytes=458296 combine_payload_bytes=903053312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=903511608 ingress_bytes=978550784 link_bytes=978550784 token_bytes=1806564920 weight_movement_bytes=75497472 total_nvlink_bytes=1882062392 weight_to_token_ratio=0.041791 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.240940136e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113350 payload_rows_remote=96966 dup_rows=17722 dup_rows_remote=17590 dedup_ratio=0.135208 router_tokens_per_expert=[0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 85, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 9, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5824, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7633, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2854, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 0, 8174] payload_rows_per_dst=[16384, 16384, 16148, 16380, 16384, 15270, 28, 16372] dup_rows_per_dst=[132, 5934, 6, 11646, 0, 0, 0, 4] combine_rows_per_src=[16384, 16384, 16148, 16380, 16384, 15270, 28, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=935297024 dispatch_meta_bytes=458752 combine_payload_bytes=935297024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=935755776 ingress_bytes=1010794496 link_bytes=1010794496 token_bytes=1871052800 weight_movement_bytes=75497472 total_nvlink_bytes=1946550272 weight_to_token_ratio=0.040350 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.281829701e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=936869888 dispatch_meta_bytes=458752 combine_payload_bytes=936869888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937328640 ingress_bytes=1012367360 link_bytes=1012367360 token_bytes=1874198528 weight_movement_bytes=75497472 total_nvlink_bytes=1949696000 weight_to_token_ratio=0.040283 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283824314e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=794345472 dispatch_meta_bytes=458224 combine_payload_bytes=794345472 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=794803696 ingress_bytes=945340416 link_bytes=945340416 token_bytes=1589149168 weight_movement_bytes=150994944 total_nvlink_bytes=1740144112 weight_to_token_ratio=0.095016 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65152 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.198824714e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=120542 payload_rows_remote=104160 dup_rows=10530 dup_rows_remote=10530 dedup_ratio=0.080338 router_tokens_per_expert=[27, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 4, 0, 20, 8185, 0, 5150, 0, 6, 0, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 5, 0, 0, 0, 0, 0, 1, 0, 0, 0, 6, 1, 6, 5, 0, 6, 0, 8, 0, 0, 0, 2, 0, 0, 0, 0, 63, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8119, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 6, 0, 0, 0, 0, 5, 1, 0, 0, 0, 0, 8185, 0, 0, 0, 0, 0, 0, 4386, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6811] payload_rows_per_dst=[16322, 16384, 16382, 16384, 16280, 16384, 8784, 13622] dup_rows_per_dst=[54, 10334, 0, 56, 84, 0, 0, 2] combine_rows_per_src=[16322, 16384, 16382, 16384, 16280, 16384, 8784, 13622] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=102924 payload_rows_remote=86566 dup_rows=28148 dup_rows_remote=25228 dedup_ratio=0.214752 router_tokens_per_expert=[0, 0, 0, 0, 8140, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 74, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 20, 0, 0, 0, 0, 0, 0, 16, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2525, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7050, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 6759, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 8175] payload_rows_per_dst=[15980, 116, 16384, 5050, 16384, 16290, 16362, 16358] dup_rows_per_dst=[44, 0, 82, 0, 14, 14100, 10988, 2920] combine_rows_per_src=[15980, 116, 16384, 5050, 16384, 16290, 16362, 16358] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114788 payload_rows_remote=98424 dup_rows=16284 dup_rows_remote=184 dedup_ratio=0.124237 router_tokens_per_expert=[0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 27, 0, 0, 0, 0, 0, 0, 6, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2388, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 5875, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 8056, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 8154] payload_rows_per_dst=[16384, 16384, 16384, 4776, 16384, 11778, 16364, 16334] dup_rows_per_dst=[98, 0, 86, 0, 0, 0, 16100, 0] combine_rows_per_src=[16384, 16384, 16384, 4776, 16384, 11778, 16364, 16334] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118354 payload_rows_remote=105222 dup_rows=12718 dup_rows_remote=12718 dedup_ratio=0.097031 router_tokens_per_expert=[0, 0, 0, 0, 8173, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 6, 0, 0, 0, 0, 0, 0, 4, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2193, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6565, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 7633, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 8162] payload_rows_per_dst=[16378, 16384, 16384, 7008, 16384, 13132, 16358, 16326] dup_rows_per_dst=[30, 0, 22, 900, 0, 0, 11766, 0] combine_rows_per_src=[16378, 16384, 16384, 7008, 16384, 13132, 16358, 16326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128868 payload_rows_remote=112484 dup_rows=2204 dup_rows_remote=32 dedup_ratio=0.016815 router_tokens_per_expert=[0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1086, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7516, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 7782, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 8185] payload_rows_per_dst=[16362, 16384, 16384, 16384, 16384, 15032, 15564, 16374] dup_rows_per_dst=[10, 0, 8, 2172, 6, 0, 0, 8] combine_rows_per_src=[16362, 16384, 16384, 16384, 16384, 15032, 15564, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=709148672 dispatch_meta_bytes=447176 combine_payload_bytes=709148672 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=709595848 ingress_bytes=784646144 link_bytes=784646144 token_bytes=1418744520 weight_movement_bytes=75497472 total_nvlink_bytes=1494241992 weight_to_token_ratio=0.053214 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.950417579e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=853278720 dispatch_meta_bytes=458760 combine_payload_bytes=853278720 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=853737480 ingress_bytes=928776192 link_bytes=928776192 token_bytes=1707016200 weight_movement_bytes=75497472 total_nvlink_bytes=1782513672 weight_to_token_ratio=0.044228 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177818947e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=806289408 dispatch_meta_bytes=394432 combine_payload_bytes=806289408 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=806683840 ingress_bytes=1108279296 link_bytes=1108279296 token_bytes=1612973248 weight_movement_bytes=301989888 total_nvlink_bytes=1914963136 weight_to_token_ratio=0.187226 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=42624 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.405454149e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128750 payload_rows_remote=112762 dup_rows=2322 dup_rows_remote=2238 dedup_ratio=0.017715 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1109, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 7463, 0, 0, 0, 0, 7992, 0, 7957, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 14926, 16384, 15988, 15918, 16384, 16382] dup_rows_per_dst=[2218, 12, 0, 0, 84, 8, 0, 0] combine_rows_per_src=[16384, 16384, 14926, 16384, 15988, 15918, 16384, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=126450 payload_rows_remote=110066 dup_rows=4622 dup_rows_remote=4604 dedup_ratio=0.035263 router_tokens_per_expert=[0, 0, 8192, 0, 0, 8192, 0, 0, 0, 1931, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 6039, 0, 0, 0, 0, 8144, 0, 8167, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 385, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8094, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 12078, 16378, 16320, 16334, 16384, 16188] dup_rows_per_dst=[3862, 18, 0, 0, 742, 0, 0, 0] combine_rows_per_src=[16384, 16384, 12078, 16378, 16320, 16334, 16384, 16188] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=861978624 dispatch_meta_bytes=471760 combine_payload_bytes=861978624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=862450384 ingress_bytes=937476096 link_bytes=937476096 token_bytes=1724429008 weight_movement_bytes=75497472 total_nvlink_bytes=1799926480 weight_to_token_ratio=0.043781 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=57472 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.188851649e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=119684 payload_rows_remote=103300 dup_rows=11388 dup_rows_remote=2250 dedup_ratio=0.086884 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 1330, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 257, 0, 0, 0, 0, 806, 0, 0, 5128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8184, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 564, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 145, 0, 0, 0, 0, 0, 0, 8164, 0, 8186, 0] payload_rows_per_dst=[16384, 16384, 16380, 12354, 16384, 9694, 16372, 15732] dup_rows_per_dst=[9138, 204, 248, 28, 24, 860, 596, 290] combine_rows_per_src=[16384, 16384, 16380, 12354, 16384, 9694, 16372, 15732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=921468928 dispatch_meta_bytes=450064 combine_payload_bytes=921468928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=921918992 ingress_bytes=996966400 link_bytes=996966400 token_bytes=1843387920 weight_movement_bytes=75497472 total_nvlink_bytes=1918885392 weight_to_token_ratio=0.040956 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.044118 gemm_active_groups=10 gemm_peak_group_rows=42752 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.264293729e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=901660672 dispatch_meta_bytes=458680 combine_payload_bytes=901660672 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=902119352 ingress_bytes=977158144 link_bytes=977158144 token_bytes=1803780024 weight_movement_bytes=75497472 total_nvlink_bytes=1879277496 weight_to_token_ratio=0.041855 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.239174072e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=923746304 dispatch_meta_bytes=460000 combine_payload_bytes=923746304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=924206304 ingress_bytes=999243776 link_bytes=999243776 token_bytes=1847952608 weight_movement_bytes=75497472 total_nvlink_bytes=1923450080 weight_to_token_ratio=0.040855 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=62848 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.267181762e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=846233600 dispatch_meta_bytes=422200 combine_payload_bytes=846233600 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=846655800 ingress_bytes=1072726016 link_bytes=1072726016 token_bytes=1692889400 weight_movement_bytes=226492416 total_nvlink_bytes=1919381816 weight_to_token_ratio=0.133790 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=41600 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.360367586e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113812 payload_rows_remote=97506 dup_rows=17260 dup_rows_remote=3008 dedup_ratio=0.131683 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 1122, 0, 0, 0, 8189, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 799, 0, 0, 0, 0, 3769, 0, 0, 1859, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 194, 0, 0, 0, 0, 0, 0, 8150, 0, 8182, 0] payload_rows_per_dst=[2270, 16384, 16366, 13354, 16384, 16384, 16364, 16306] dup_rows_per_dst=[0, 10, 0, 2020, 0, 978, 0, 14252] combine_rows_per_src=[2270, 16384, 16366, 13354, 16384, 16384, 16364, 16306] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114012 payload_rows_remote=97850 dup_rows=17060 dup_rows_remote=17036 dedup_ratio=0.130157 router_tokens_per_expert=[0, 8186, 0, 0, 0, 64, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 0, 0, 8070, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 8073, 0, 0, 0, 0, 2, 6, 5, 0, 0, 6, 0, 1377, 1, 6999, 0, 0, 0, 6, 0, 21, 0, 6, 0, 6, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 6, 8168, 0, 0, 0, 0, 0, 0, 7998, 0, 0, 0, 160, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 258, 0, 7919, 0] payload_rows_per_dst=[16372, 16140, 16162, 16338, 16348, 16284, 15852, 516] dup_rows_per_dst=[16464, 0, 24, 494, 12, 34, 2, 30] combine_rows_per_src=[16372, 16140, 16162, 16338, 16348, 16284, 15852, 516] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=115184 payload_rows_remote=98824 dup_rows=15888 dup_rows_remote=15888 dedup_ratio=0.121216 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 309, 0, 0, 0, 8192, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 606, 0, 0, 0, 0, 6445, 0, 0, 402, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 0, 446, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 35, 0, 0, 0, 0, 0, 0, 8141, 0, 8180, 0] payload_rows_per_dst=[644, 16384, 16362, 16384, 16384, 16384, 16360, 16282] dup_rows_per_dst=[0, 24, 0, 14906, 0, 888, 0, 70] combine_rows_per_src=[644, 16384, 16362, 16384, 16384, 16384, 16360, 16282] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=116192 payload_rows_remote=99808 dup_rows=14880 dup_rows_remote=13686 dedup_ratio=0.113525 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 347, 0, 0, 0, 8192, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 466, 0, 0, 0, 0, 6126, 0, 0, 639, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 598, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 39, 0, 0, 0, 0, 0, 0, 8174, 0, 8187, 0] payload_rows_per_dst=[1826, 16384, 16380, 16112, 16384, 16384, 16374, 16348] dup_rows_per_dst=[2, 4, 0, 13602, 0, 1194, 0, 78] combine_rows_per_src=[1826, 16384, 16380, 16112, 16384, 16384, 16374, 16348] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 2, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121700 payload_rows_remote=114686 dup_rows=9372 dup_rows_remote=9372 dedup_ratio=0.071503 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3589, 0, 0, 0, 8192, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 614, 0, 0, 0, 0, 1303, 0, 0, 1590, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 513, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 585, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16382, 7014, 16384, 16384, 16384, 16384] dup_rows_per_dst=[7178, 2, 0, 0, 0, 1022, 0, 1170] combine_rows_per_src=[16384, 16384, 16382, 7014, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=798769152 dispatch_meta_bytes=402056 combine_payload_bytes=798769152 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=799171208 ingress_bytes=1025261568 link_bytes=1025261568 token_bytes=1597940360 weight_movement_bytes=226492416 total_nvlink_bytes=1824432776 weight_to_token_ratio=0.141740 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=57216 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.300175985e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122208 payload_rows_remote=105824 dup_rows=8864 dup_rows_remote=8864 dedup_ratio=0.067627 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1533, 44, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 2816, 0, 8192, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 30, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 0, 3805, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8123, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 19, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 7670, 16372, 16248, 16382] dup_rows_per_dst=[3154, 0, 5634, 40, 0, 0, 36, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 7670, 16372, 16248, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127670 payload_rows_remote=114248 dup_rows=3402 dup_rows_remote=3400 dedup_ratio=0.025955 router_tokens_per_expert=[0, 0, 8192, 0, 0, 1399, 38, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 186, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 47, 0, 0, 0, 0, 8178, 0, 0, 0, 0, 0, 0, 0, 0, 6665, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7970, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 91, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 13422, 16356, 15972, 16384] dup_rows_per_dst=[2874, 0, 374, 2, 2, 0, 150, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 13422, 16356, 15972, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=809566208 dispatch_meta_bytes=458848 combine_payload_bytes=809566208 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=810025056 ingress_bytes=960561152 link_bytes=960561152 token_bytes=1619591264 weight_movement_bytes=150994944 total_nvlink_bytes=1770586208 weight_to_token_ratio=0.093230 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.218126750e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=801587200 dispatch_meta_bytes=459544 combine_payload_bytes=801587200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=802046744 ingress_bytes=877084672 link_bytes=877084672 token_bytes=1603633944 weight_movement_bytes=75497472 total_nvlink_bytes=1679131416 weight_to_token_ratio=0.047079 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.112266823e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=817627136 dispatch_meta_bytes=453976 combine_payload_bytes=817627136 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=818081112 ingress_bytes=1119617024 link_bytes=1119617024 token_bytes=1635708248 weight_movement_bytes=301989888 total_nvlink_bytes=1937698136 weight_to_token_ratio=0.184623 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=61952 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.419831984e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113940 payload_rows_remote=97556 dup_rows=17132 dup_rows_remote=17132 dedup_ratio=0.130707 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 12, 82, 0, 0, 0, 0, 0, 0, 0, 0, 8115, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 28, 16378, 16358, 16372, 15652] dup_rows_per_dst=[0, 0, 17116, 0, 16, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 28, 16378, 16358, 16372, 15652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 4, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=939507712 dispatch_meta_bytes=496232 combine_payload_bytes=939507712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=940003944 ingress_bytes=1015005184 link_bytes=1015005184 token_bytes=1879511656 weight_movement_bytes=75497472 total_nvlink_bytes=1955009128 weight_to_token_ratio=0.040169 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1270 gemm_tail_pad_fraction=0.047794 gemm_active_groups=10 gemm_peak_group_rows=22016 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287169446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=866910208 dispatch_meta_bytes=458752 combine_payload_bytes=866910208 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=867368960 ingress_bytes=1017905152 link_bytes=1017905152 token_bytes=1734279168 weight_movement_bytes=150994944 total_nvlink_bytes=1885274112 weight_to_token_ratio=0.087065 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.290847014e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=935919616 dispatch_meta_bytes=470592 combine_payload_bytes=935919616 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=936390208 ingress_bytes=1162412032 link_bytes=1162412032 token_bytes=1872309824 weight_movement_bytes=226492416 total_nvlink_bytes=2098802240 weight_to_token_ratio=0.120970 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=42624 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.474102078e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=799178752 dispatch_meta_bytes=458752 combine_payload_bytes=799178752 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=799637504 ingress_bytes=950173696 link_bytes=950173696 token_bytes=1598816256 weight_movement_bytes=150994944 total_nvlink_bytes=1749811200 weight_to_token_ratio=0.094442 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.204953994e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=111742 payload_rows_remote=95358 dup_rows=19330 dup_rows_remote=12030 dedup_ratio=0.147476 router_tokens_per_expert=[0, 4, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 46, 0, 0, 0, 0, 0, 0, 0, 9, 0, 0, 0, 0, 0, 20, 25, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16338, 16384, 4546, 16384, 16286, 16342, 9078, 16384] dup_rows_per_dst=[2, 0, 0, 150, 11862, 16, 0, 7300] combine_rows_per_src=[16338, 16384, 4546, 16384, 16286, 16342, 9078, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130678 payload_rows_remote=114330 dup_rows=394 dup_rows_remote=394 dedup_ratio=0.003006 router_tokens_per_expert=[0, 2, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 141, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 39, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8049, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8158, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 8174, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16098, 16316, 16348, 16380] dup_rows_per_dst=[4, 0, 0, 376, 14, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16098, 16316, 16348, 16380] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=99532 payload_rows_remote=97030 dup_rows=31540 dup_rows_remote=31540 dedup_ratio=0.240631 router_tokens_per_expert=[0, 0, 0, 1, 0, 1, 0, 0, 8186, 0, 0, 22, 0, 0, 0, 0, 0, 2, 0, 0, 3, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 1, 0, 0, 0, 0, 0, 1250, 0, 0, 0, 0, 4555, 0, 3050, 0, 0, 0, 0, 0, 0, 4337, 0, 0, 0, 0, 2, 2665, 0, 5, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 104, 0, 8186, 0, 0, 0, 8181, 1, 0, 0, 3710, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 6, 6, 4879, 5, 14, 0, 8158, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16374, 16380, 2502, 16372, 5342, 16384, 16380, 9798] dup_rows_per_dst=[46, 4, 0, 23882, 10, 210, 7366, 22] combine_rows_per_src=[16374, 16380, 2502, 16372, 5342, 16384, 16380, 9798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130772 payload_rows_remote=114406 dup_rows=300 dup_rows_remote=300 dedup_ratio=0.002289 router_tokens_per_expert=[0, 1, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 143, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 8051, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16106, 16366, 16380, 16384] dup_rows_per_dst=[2, 0, 0, 294, 4, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16106, 16366, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130932 payload_rows_remote=114548 dup_rows=140 dup_rows_remote=12 dedup_ratio=0.001068 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 1, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 60, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 10, 0, 0, 0, 0, 0, 0, 0, 0, 8130, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 1, 0, 8189, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16382, 16384, 16270, 16364, 16380, 16384] dup_rows_per_dst=[2, 0, 0, 128, 10, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16382, 16384, 16270, 16364, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=781172736 dispatch_meta_bytes=429552 combine_payload_bytes=781172736 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=781602288 ingress_bytes=1007665152 link_bytes=1007665152 token_bytes=1562775024 weight_movement_bytes=226492416 total_nvlink_bytes=1789267440 weight_to_token_ratio=0.144930 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=61952 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.277861252e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=936591360 dispatch_meta_bytes=458896 combine_payload_bytes=936591360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937050256 ingress_bytes=1012088832 link_bytes=1012088832 token_bytes=1873641616 weight_movement_bytes=75497472 total_nvlink_bytes=1949139088 weight_to_token_ratio=0.040295 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=61824 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.283471101e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=118590 payload_rows_remote=102206 dup_rows=12482 dup_rows_remote=12482 dedup_ratio=0.095230 router_tokens_per_expert=[0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 49, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 11, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6197, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 7563, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 2586, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 0, 0, 8181] payload_rows_per_dst=[16384, 16384, 16384, 16382, 16384, 15126, 5172, 16374] dup_rows_per_dst=[66, 0, 24, 12390, 0, 0, 0, 2] combine_rows_per_src=[16384, 16384, 16384, 16382, 16384, 15126, 5172, 16374] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125842 payload_rows_remote=109458 dup_rows=5230 dup_rows_remote=5230 dedup_ratio=0.039902 router_tokens_per_expert=[0, 0, 0, 0, 8184, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 38, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 8192, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2580, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 6980, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 6793, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16384, 13960, 13586, 16376] dup_rows_per_dst=[60, 0, 10, 5160, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16384, 13960, 13586, 16376] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117204 payload_rows_remote=100820 dup_rows=13868 dup_rows_remote=13786 dedup_ratio=0.105804 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8173, 0, 0, 0, 0, 3773, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 73, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4518, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8039, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 10214, 9036, 16384, 16078, 16340] dup_rows_per_dst=[82, 13568, 156, 18, 0, 44, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 10214, 9036, 16384, 16078, 16340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=937213952 dispatch_meta_bytes=458824 combine_payload_bytes=937213952 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=937672776 ingress_bytes=1088208896 link_bytes=1088208896 token_bytes=1874886728 weight_movement_bytes=150994944 total_nvlink_bytes=2025881672 weight_to_token_ratio=0.080536 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.380002057e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=938377216 dispatch_meta_bytes=458240 combine_payload_bytes=938377216 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=938835456 ingress_bytes=1013874688 link_bytes=1013874688 token_bytes=1877212672 weight_movement_bytes=75497472 total_nvlink_bytes=1952710144 weight_to_token_ratio=0.040218 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.053309 gemm_active_groups=7 gemm_peak_group_rows=53120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.285735818e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=794869760 dispatch_meta_bytes=514280 combine_payload_bytes=794869760 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=795384040 ingress_bytes=1021362176 link_bytes=1021362176 token_bytes=1590253800 weight_movement_bytes=226492416 total_nvlink_bytes=1816746216 weight_to_token_ratio=0.142425 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.055147 gemm_active_groups=5 gemm_peak_group_rows=52736 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.295231007e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=837271552 dispatch_meta_bytes=458752 combine_payload_bytes=837271552 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=837730304 ingress_bytes=1063763968 link_bytes=1063763968 token_bytes=1675001856 weight_movement_bytes=226492416 total_nvlink_bytes=1901494272 weight_to_token_ratio=0.135219 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=57472 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.349002448e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=896679936 dispatch_meta_bytes=458752 combine_payload_bytes=896679936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=897138688 ingress_bytes=972177408 link_bytes=972177408 token_bytes=1793818624 weight_movement_bytes=75497472 total_nvlink_bytes=1869316096 weight_to_token_ratio=0.042088 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.232857798e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=825917440 dispatch_meta_bytes=458424 combine_payload_bytes=825917440 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=826375864 ingress_bytes=976912384 link_bytes=976912384 token_bytes=1652293304 weight_movement_bytes=150994944 total_nvlink_bytes=1803288248 weight_to_token_ratio=0.091385 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.238862414e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114236 payload_rows_remote=97852 dup_rows=16836 dup_rows_remote=16468 dedup_ratio=0.128448 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8144, 0, 0, 0, 0, 6090, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 84, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8186, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 2113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8145, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16016, 0, 16384, 16384, 16300, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 156, 4226, 0, 12086, 368] combine_rows_per_src=[16016, 0, 16384, 16384, 16300, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 2, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=121822 payload_rows_remote=105438 dup_rows=9250 dup_rows_remote=5532 dedup_ratio=0.070572 router_tokens_per_expert=[0, 0, 0, 0, 5, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 8134, 0, 0, 0, 0, 5487, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1523, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 1247, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 8183, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 7252, 16384, 16384, 16284, 16366, 16384, 16384] dup_rows_per_dst=[22, 0, 0, 3016, 2494, 0, 3718, 0] combine_rows_per_src=[16384, 7252, 16384, 16384, 16284, 16366, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=801603584 dispatch_meta_bytes=457280 combine_payload_bytes=801603584 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=802060864 ingress_bytes=1028096000 link_bytes=1028096000 token_bytes=1603664448 weight_movement_bytes=226492416 total_nvlink_bytes=1830156864 weight_to_token_ratio=0.141234 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=65408 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.303770443e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125594 payload_rows_remote=109210 dup_rows=5478 dup_rows_remote=5478 dedup_ratio=0.041794 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 1, 0, 0, 5494, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 910, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8180, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1840, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 10988, 16384, 16384, 16376, 16384, 16310, 16384] dup_rows_per_dst=[0, 2, 0, 1796, 3680, 0, 0, 0] combine_rows_per_src=[16384, 10988, 16384, 16384, 16376, 16384, 16310, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124670 payload_rows_remote=108288 dup_rows=6402 dup_rows_remote=6014 dedup_ratio=0.048843 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 8187, 0, 0, 1, 0, 5034, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 198, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8187, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3004, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8155, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 1, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 10068, 16384, 16382, 16374, 16384, 16310, 16384] dup_rows_per_dst=[2, 2, 0, 388, 6008, 0, 0, 2] combine_rows_per_src=[16384, 10068, 16384, 16382, 16374, 16384, 16310, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=113562 payload_rows_remote=97334 dup_rows=17510 dup_rows_remote=17510 dedup_ratio=0.133591 router_tokens_per_expert=[0, 0, 0, 0, 141, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 29, 0, 0, 8185, 0, 0, 159, 6, 8055, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8107, 0, 0, 1, 0, 0, 6, 0, 1, 0, 31, 1010, 1, 0, 0, 0, 0, 0, 0, 0, 0, 7037, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 7596, 0, 0, 6, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 7, 0, 1, 0, 23, 561, 0, 176, 0, 0, 0, 8164, 0, 0, 0, 6, 6, 0, 0, 10, 0, 8007, 0, 0] payload_rows_per_dst=[16370, 16384, 16228, 15462, 15266, 16338, 1468, 16046] dup_rows_per_dst=[276, 16062, 0, 698, 362, 20, 80, 12] combine_rows_per_src=[16370, 16384, 16228, 15462, 15266, 16338, 1468, 16046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 4] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=863748096 dispatch_meta_bytes=443880 combine_payload_bytes=863748096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=864191976 ingress_bytes=939245568 link_bytes=939245568 token_bytes=1727940072 weight_movement_bytes=75497472 total_nvlink_bytes=1803437544 weight_to_token_ratio=0.043692 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.047794 gemm_active_groups=8 gemm_peak_group_rows=57472 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.191095589e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125286 payload_rows_remote=108902 dup_rows=5786 dup_rows_remote=5786 dedup_ratio=0.044144 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2390, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 87, 0, 0, 0, 0, 799, 0, 0, 4434, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 324, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 178, 0, 0, 0, 0, 0, 0, 8173, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 10636, 16384, 16384, 16384, 16346] dup_rows_per_dst=[4780, 0, 0, 4, 0, 646, 0, 356] combine_rows_per_src=[16384, 16384, 16384, 10636, 16384, 16384, 16384, 16346] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122456 payload_rows_remote=106072 dup_rows=8616 dup_rows_remote=8616 dedup_ratio=0.065735 router_tokens_per_expert=[0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3431, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 472, 0, 0, 0, 0, 2679, 0, 0, 762, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 584, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 293, 0, 0, 0, 0, 0, 0, 8163, 0, 8192, 0] payload_rows_per_dst=[16384, 16384, 16384, 7826, 16384, 16384, 16384, 16326] dup_rows_per_dst=[6862, 0, 0, 0, 0, 1168, 0, 586] combine_rows_per_src=[16384, 16384, 16384, 7826, 16384, 16384, 16384, 16326] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125374 payload_rows_remote=108990 dup_rows=5698 dup_rows_remote=5698 dedup_ratio=0.043472 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8186, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16380, 13620, 13454, 16384] dup_rows_per_dst=[0, 4300, 0, 1392, 6, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16380, 13620, 13454, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=894648320 dispatch_meta_bytes=458752 combine_payload_bytes=894648320 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=895107072 ingress_bytes=1045643264 link_bytes=1045643264 token_bytes=1789755392 weight_movement_bytes=150994944 total_nvlink_bytes=1940750336 weight_to_token_ratio=0.084366 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.326022844e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=887095296 dispatch_meta_bytes=457208 combine_payload_bytes=887095296 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=887552504 ingress_bytes=1113587712 link_bytes=1113587712 token_bytes=1774647800 weight_movement_bytes=226492416 total_nvlink_bytes=2001140216 weight_to_token_ratio=0.127627 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=61312 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.412185968e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=797360128 dispatch_meta_bytes=459376 combine_payload_bytes=797360128 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=797819504 ingress_bytes=872857600 link_bytes=872857600 token_bytes=1595179632 weight_movement_bytes=75497472 total_nvlink_bytes=1670677104 weight_to_token_ratio=0.047329 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.106906301e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=892125184 dispatch_meta_bytes=458752 combine_payload_bytes=892125184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=892583936 ingress_bytes=967622656 link_bytes=967622656 token_bytes=1784709120 weight_movement_bytes=75497472 total_nvlink_bytes=1860206592 weight_to_token_ratio=0.042302 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.227081731e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=868941824 dispatch_meta_bytes=458752 combine_payload_bytes=868941824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=869400576 ingress_bytes=944439296 link_bytes=944439296 token_bytes=1738342400 weight_movement_bytes=75497472 total_nvlink_bytes=1813839872 weight_to_token_ratio=0.043431 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.197681967e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=892846080 dispatch_meta_bytes=458752 combine_payload_bytes=892846080 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=893304832 ingress_bytes=968343552 link_bytes=968343552 token_bytes=1786150912 weight_movement_bytes=75497472 total_nvlink_bytes=1861648384 weight_to_token_ratio=0.042268 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.227995929e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=123522 payload_rows_remote=107138 dup_rows=7550 dup_rows_remote=7548 dedup_ratio=0.057602 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8160, 8192, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8188, 0, 0, 7, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 24, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8180, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 12114, 16376, 16384, 13112, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 4214, 14, 38, 3256, 26, 2] combine_rows_per_src=[16384, 12114, 16376, 16384, 13112, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130994 payload_rows_remote=114610 dup_rows=78 dup_rows_remote=40 dedup_ratio=0.000595 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 5, 0, 8167, 8192, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 8182, 0, 0, 11, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8188, 8174, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 18, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16346, 16380, 16384, 16356, 16376, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 22, 18, 0, 38, 0] combine_rows_per_src=[16384, 16346, 16380, 16384, 16356, 16376, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=4 unused_prefetch_slots=28 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=2 remote_demand_peak=2 slot_utilization=0.125000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 4, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131064 payload_rows_remote=114680 dup_rows=8 dup_rows_remote=8 dedup_ratio=0.000061 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16376, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 8, 0, 0, 0] combine_rows_per_src=[16384, 16376, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=877674496 dispatch_meta_bytes=458744 combine_payload_bytes=877674496 prefetch_bytes=150994944 reduce_bytes=0 egress_bytes=878133240 ingress_bytes=1028669440 link_bytes=1028669440 token_bytes=1755807736 weight_movement_bytes=150994944 total_nvlink_bytes=1906802680 weight_to_token_ratio=0.085997 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.304497646e-04 prefetch_bytes_per_gemm_flop=1.914828431e-05 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131062 payload_rows_remote=114680 dup_rows=10 dup_rows_remote=10 dedup_ratio=0.000076 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 1, 0, 8188, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8191, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16378, 16384, 16384, 16382, 16382, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 2, 0, 6, 0] combine_rows_per_src=[16384, 16378, 16384, 16384, 16382, 16382, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=124818 payload_rows_remote=112636 dup_rows=6254 dup_rows_remote=6254 dedup_ratio=0.047714 router_tokens_per_expert=[0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 10, 0, 0, 8185, 156, 0, 8095, 8183, 0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 0, 6086, 0, 0, 0, 0, 7462, 1, 1, 0, 9, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 0, 1567, 5, 0, 0, 0, 0, 0, 0, 0, 6, 8181, 0, 7942, 8179, 0, 0, 2, 1, 0, 0, 0, 0, 35, 0, 0, 1, 0, 1385, 0, 0, 6, 0, 0, 0, 0, 6, 0, 6, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0] payload_rows_per_dst=[16384, 16264, 12182, 14944, 16382, 15898, 16382, 16382] dup_rows_per_dst=[0, 258, 0, 2, 3128, 4, 2862, 0] combine_rows_per_src=[16384, 16264, 12182, 14944, 16382, 15898, 16382, 16382] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=938885120 dispatch_meta_bytes=458600 combine_payload_bytes=938885120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939343720 ingress_bytes=1014382592 link_bytes=1014382592 token_bytes=1878228840 weight_movement_bytes=75497472 total_nvlink_bytes=1953726312 weight_to_token_ratio=0.040196 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.047794 gemm_active_groups=9 gemm_peak_group_rows=64128 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286379912e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122456 payload_rows_remote=106072 dup_rows=8616 dup_rows_remote=8616 dedup_ratio=0.065735 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 36, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8161, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8177, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8190, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 7862, 16324, 16354, 16380, 16384] dup_rows_per_dst=[0, 0, 8606, 10, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 7862, 16324, 16354, 16380, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131006 payload_rows_remote=114666 dup_rows=66 dup_rows_remote=54 dedup_ratio=0.000504 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 20, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8169, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 16340, 16362, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 50, 12, 0, 4, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 16340, 16362, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=5 export_slots=5 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 2] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114604 payload_rows_remote=98220 dup_rows=16468 dup_rows_remote=15984 dedup_ratio=0.125641 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8168, 0, 2, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8080, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8190, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 131, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16112, 16372, 16384, 16352, 16384, 262, 16354] dup_rows_per_dst=[484, 0, 15952, 32, 0, 0, 0, 0] combine_rows_per_src=[16384, 16112, 16372, 16384, 16352, 16384, 262, 16354] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458752 combine_payload_bytes=939458560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939917312 ingress_bytes=1014956032 link_bytes=1014956032 token_bytes=1879375872 weight_movement_bytes=75497472 total_nvlink_bytes=1954873344 weight_to_token_ratio=0.040172 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=64896 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.287107114e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939458560 dispatch_meta_bytes=458760 combine_payload_bytes=939458560 prefetch_bytes=0 reduce_bytes=377487360 egress_bytes=939917320 ingress_bytes=1316945920 link_bytes=1316945920 token_bytes=1879375880 weight_movement_bytes=377487360 total_nvlink_bytes=2256863240 weight_to_token_ratio=0.200858 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=64000 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.670072801e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=922714112 dispatch_meta_bytes=475560 combine_payload_bytes=922714112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=923189672 ingress_bytes=998211584 link_bytes=998211584 token_bytes=1845903784 weight_movement_bytes=75497472 total_nvlink_bytes=1921401256 weight_to_token_ratio=0.040900 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=63488 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265872797e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=939343872 dispatch_meta_bytes=458880 combine_payload_bytes=939343872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939802752 ingress_bytes=1014841344 link_bytes=1014841344 token_bytes=1879146624 weight_movement_bytes=75497472 total_nvlink_bytes=1954644096 weight_to_token_ratio=0.040176 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.053309 gemm_active_groups=6 gemm_peak_group_rows=56832 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286961674e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=868941824 dispatch_meta_bytes=458752 combine_payload_bytes=868941824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=869400576 ingress_bytes=944439296 link_bytes=944439296 token_bytes=1738342400 weight_movement_bytes=75497472 total_nvlink_bytes=1813839872 weight_to_token_ratio=0.043431 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.197681967e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=804618240 dispatch_meta_bytes=456816 combine_payload_bytes=804618240 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=805075056 ingress_bytes=955613184 link_bytes=955613184 token_bytes=1609693296 weight_movement_bytes=150994944 total_nvlink_bytes=1760688240 weight_to_token_ratio=0.093804 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.211852030e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130818 payload_rows_remote=114434 dup_rows=254 dup_rows_remote=254 dedup_ratio=0.001938 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8134, 0, 8, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 3, 7, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 110, 0, 0, 0, 0, 0, 8182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16284, 16384, 16258, 16382, 16376, 16384, 16366, 16384] dup_rows_per_dst=[0, 0, 8, 0, 20, 0, 226, 0] combine_rows_per_src=[16284, 16384, 16258, 16382, 16376, 16384, 16366, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127896 payload_rows_remote=111530 dup_rows=3176 dup_rows_remote=48 dedup_ratio=0.024231 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8098, 0, 9, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 14, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 6708, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8182, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1568, 0, 0, 0, 0, 0, 8179, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16214, 16384, 13416, 16364, 16384, 16384, 16366, 16384] dup_rows_per_dst=[0, 0, 44, 0, 4, 0, 3128, 0] combine_rows_per_src=[16214, 16384, 13416, 16364, 16384, 16384, 16366, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19796063096 prefetch_bytes=679477248 reduce_bytes=1283457024 weight_movement_bytes=1962934272 egress_bytes=9900700536 ingress_bytes=11858296832 link_bytes=11858296832 total_nvlink_bytes=21758997368 weight_to_token_ratio=0.099158 prefetch_share_of_ingress=0.057300 dedup_ratio=0.107115 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=17 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051624 gemm_inner_pad_upper_bound=10287 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.253165797e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=128996 payload_rows_remote=112612 dup_rows=2076 dup_rows_remote=2076 dedup_ratio=0.015839 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8154, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 7191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1038, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16308, 16384, 14384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 2076, 0] combine_rows_per_src=[16308, 16384, 14384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=127864 payload_rows_remote=111480 dup_rows=3208 dup_rows_remote=3208 dedup_ratio=0.024475 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8166, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 6614, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1602, 0, 0, 0, 0, 0, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16334, 16384, 13230, 16382, 16384, 16384, 16382, 16384] dup_rows_per_dst=[0, 0, 4, 0, 0, 0, 3204, 0] combine_rows_per_src=[16334, 16384, 13230, 16382, 16384, 16384, 16382, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=937443328 dispatch_meta_bytes=458752 combine_payload_bytes=937443328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=937902080 ingress_bytes=1012940800 link_bytes=1012940800 token_bytes=1875345408 weight_movement_bytes=75497472 total_nvlink_bytes=1950842880 weight_to_token_ratio=0.040258 gemm_rows_defined=65664 gemm_rows_charged=69632 gemm_tail_pad_rows=3968 gemm_inner_pad_upper_bound=381 gemm_tail_pad_fraction=0.056985 gemm_active_groups=3 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.284551517e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=114910 payload_rows_remote=114600 dup_rows=16162 dup_rows_remote=16162 dedup_ratio=0.123306 router_tokens_per_expert=[0, 1, 0, 0, 8176, 0, 0, 0, 0, 0, 6, 0, 8186, 0, 0, 0, 8185, 0, 0, 131, 0, 0, 1, 0, 0, 1, 0, 4, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 149, 0, 0, 0, 0, 0, 0, 0, 0, 22, 0, 0, 0, 0, 0, 0, 8165, 1, 0, 0, 0, 0, 1, 0, 6, 0, 1, 0, 0, 8184, 0, 6, 1, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0, 0, 8176, 0, 0, 0, 0, 0, 0, 10, 0, 0, 0, 0, 0, 8016, 0, 2, 6, 0, 0, 8050, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 31, 0, 0, 0] payload_rows_per_dst=[16384, 16384, 310, 16372, 16382, 16364, 16342, 16372] dup_rows_per_dst=[2, 272, 0, 4, 16, 0, 15826, 42] combine_rows_per_src=[16384, 16384, 310, 16372, 16382, 16364, 16342, 16372] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=125560 payload_rows_remote=109176 dup_rows=5512 dup_rows_remote=714 dedup_ratio=0.042053 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 5604, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 357, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2399, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8024, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 11208, 16384, 16384, 16384, 16384, 16048, 16384] dup_rows_per_dst=[0, 0, 0, 714, 4798, 0, 0, 0] combine_rows_per_src=[16384, 11208, 16384, 16384, 16384, 16384, 16048, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=913653760 dispatch_meta_bytes=446312 combine_payload_bytes=913653760 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=914100072 ingress_bytes=1215643648 link_bytes=1215643648 token_bytes=1827753832 weight_movement_bytes=301989888 total_nvlink_bytes=2129743720 weight_to_token_ratio=0.165225 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.049632 gemm_active_groups=7 gemm_peak_group_rows=52864 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.541607260e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=117994 payload_rows_remote=101610 dup_rows=13078 dup_rows_remote=308 dedup_ratio=0.099777 router_tokens_per_expert=[0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8181, 0, 0, 0, 0, 6396, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 157, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8189, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1742, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8103, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0] payload_rows_per_dst=[16384, 16384, 16384, 16384, 3484, 16384, 16206, 16384] dup_rows_per_dst=[0, 12770, 0, 308, 0, 0, 0, 0] combine_rows_per_src=[16384, 16384, 16384, 16384, 3484, 16384, 16206, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=19292019176 prefetch_bytes=905969664 reduce_bytes=981467136 weight_movement_bytes=1887436800 egress_bytes=9648609768 ingress_bytes=11530846208 link_bytes=11530846208 total_nvlink_bytes=21179455976 weight_to_token_ratio=0.097835 prefetch_share_of_ingress=0.078569 dedup_ratio=0.126928 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=13 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050705 gemm_inner_pad_upper_bound=10541 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.218561340e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=922517504 dispatch_meta_bytes=458752 combine_payload_bytes=922517504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=922976256 ingress_bytes=998014976 link_bytes=998014976 token_bytes=1845493760 weight_movement_bytes=75497472 total_nvlink_bytes=1920991232 weight_to_token_ratio=0.040909 gemm_rows_defined=65920 gemm_rows_charged=69632 gemm_tail_pad_rows=3712 gemm_inner_pad_upper_bound=635 gemm_tail_pad_fraction=0.053309 gemm_active_groups=5 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.265623471e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=913244160 dispatch_meta_bytes=458752 combine_payload_bytes=913244160 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=913702912 ingress_bytes=988741632 link_bytes=988741632 token_bytes=1826947072 weight_movement_bytes=75497472 total_nvlink_bytes=1902444544 weight_to_token_ratio=0.041324 gemm_rows_defined=65792 gemm_rows_charged=69632 gemm_tail_pad_rows=3840 gemm_inner_pad_upper_bound=508 gemm_tail_pad_fraction=0.055147 gemm_active_groups=4 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.253863565e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20785042896 prefetch_bytes=981467136 reduce_bytes=754974720 weight_movement_bytes=1736441856 egress_bytes=10395260368 ingress_bytes=12126224384 link_bytes=12126224384 total_nvlink_bytes=22521484752 weight_to_token_ratio=0.083543 prefetch_share_of_ingress=0.080938 dedup_ratio=0.081800 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=10 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050858 gemm_inner_pad_upper_bound=10414 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.281479951e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=938803200 dispatch_meta_bytes=523048 combine_payload_bytes=938803200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939326248 ingress_bytes=1014300672 link_bytes=1014300672 token_bytes=1878129448 weight_movement_bytes=75497472 total_nvlink_bytes=1953626920 weight_to_token_ratio=0.040198 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=53120 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286276026e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=832389120 dispatch_meta_bytes=407672 combine_payload_bytes=832389120 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=832796792 ingress_bytes=1058881536 link_bytes=1058881536 token_bytes=1665185912 weight_movement_bytes=226492416 total_nvlink_bytes=1891678328 weight_to_token_ratio=0.136016 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=38016 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.342810836e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=894369792 dispatch_meta_bytes=439560 combine_payload_bytes=894369792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=894809352 ingress_bytes=969867264 link_bytes=969867264 token_bytes=1789179144 weight_movement_bytes=75497472 total_nvlink_bytes=1864676616 weight_to_token_ratio=0.042197 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=762 gemm_tail_pad_fraction=0.051471 gemm_active_groups=6 gemm_peak_group_rows=41088 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.229928210e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21541807696 prefetch_bytes=754974720 reduce_bytes=603979776 weight_movement_bytes=1358954496 egress_bytes=10773653072 ingress_bytes=12127109120 link_bytes=12127109120 total_nvlink_bytes=22900762192 weight_to_token_ratio=0.063085 prefetch_share_of_ingress=0.062255 dedup_ratio=0.045292 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=8 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050398 gemm_inner_pad_upper_bound=10668 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.281573449e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20972844312 prefetch_bytes=981467136 reduce_bytes=754974720 weight_movement_bytes=1736441856 egress_bytes=10489165080 ingress_bytes=12220121088 link_bytes=12220121088 total_nvlink_bytes=22709286168 weight_to_token_ratio=0.082795 prefetch_share_of_ingress=0.080316 dedup_ratio=0.063858 prefetch_slots=26 unused_prefetch_slots=358 slot_utilization=0.067708 reduce_slots=10 remote_demand=13 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051317 gemm_inner_pad_upper_bound=9652 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.291402804e-04 prefetch_bytes_per_gemm_flop=1.037198734e-05 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20258029472 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=10131865504 ingress_bytes=11787108352 link_bytes=11787108352 total_nvlink_bytes=21918973856 weight_to_token_ratio=0.081989 prefetch_share_of_ingress=0.070456 dedup_ratio=0.124203 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051164 gemm_inner_pad_upper_bound=10414 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.245642713e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=131054 payload_rows_remote=114688 dup_rows=18 dup_rows_remote=18 dedup_ratio=0.000137 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8183, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 1, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 2, 16, 0, 0, 0] combine_rows_per_src=[16384, 16366, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130974 payload_rows_remote=114590 dup_rows=98 dup_rows_remote=6 dedup_ratio=0.000748 router_tokens_per_expert=[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 8143, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 3, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 47, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 8192, 8191, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16384, 16286, 16384, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 6, 92, 0, 0, 0] combine_rows_per_src=[16384, 16286, 16384, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=939524096 dispatch_meta_bytes=458824 combine_payload_bytes=939524096 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=939982920 ingress_bytes=1241513984 link_bytes=1241513984 token_bytes=1879507016 weight_movement_bytes=301989888 total_nvlink_bytes=2181496904 weight_to_token_ratio=0.160675 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=63232 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.574414488e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=938721280 dispatch_meta_bytes=458384 combine_payload_bytes=938721280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=939179664 ingress_bytes=1014218752 link_bytes=1014218752 token_bytes=1877900944 weight_movement_bytes=75497472 total_nvlink_bytes=1953398416 weight_to_token_ratio=0.040203 gemm_rows_defined=66048 gemm_rows_charged=69632 gemm_tail_pad_rows=3584 gemm_inner_pad_upper_bound=889 gemm_tail_pad_fraction=0.051471 gemm_active_groups=7 gemm_peak_group_rows=63872 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.286172140e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=122256 payload_rows_remote=105872 dup_rows=8816 dup_rows_remote=8816 dedup_ratio=0.067261 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8165, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8131, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 86, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16330, 16384, 16332, 16384, 16384, 16384, 7674, 16384] dup_rows_per_dst=[0, 0, 8742, 0, 2, 20, 52, 0] combine_rows_per_src=[16330, 16384, 16332, 16384, 16384, 16384, 7674, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=130672 payload_rows_remote=114288 dup_rows=400 dup_rows_remote=400 dedup_ratio=0.003052 router_tokens_per_expert=[0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 8174, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8010, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 200, 0, 0, 0, 0, 0, 8192, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] payload_rows_per_dst=[16348, 16384, 16020, 16384, 16384, 16384, 16384, 16384] dup_rows_per_dst=[0, 0, 0, 0, 0, 0, 400, 0] combine_rows_per_src=[16348, 16384, 16020, 16384, 16384, 16384, 16384, 16384] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=867303424 dispatch_meta_bytes=458752 combine_payload_bytes=867303424 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=867762176 ingress_bytes=1018298368 link_bytes=1018298368 token_bytes=1735065600 weight_movement_bytes=150994944 total_nvlink_bytes=1886060544 weight_to_token_ratio=0.087025 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=1016 gemm_tail_pad_fraction=0.049632 gemm_active_groups=8 gemm_peak_group_rows=65280 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.291345667e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=936247296 dispatch_meta_bytes=458752 combine_payload_bytes=936247296 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=936706048 ingress_bytes=1087242240 link_bytes=1087242240 token_bytes=1872953344 weight_movement_bytes=150994944 total_nvlink_bytes=2023948288 weight_to_token_ratio=0.080619 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=1143 gemm_tail_pad_fraction=0.045956 gemm_active_groups=9 gemm_peak_group_rows=65536 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.378776201e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=20953768296 prefetch_bytes=905969664 reduce_bytes=981467136 weight_movement_bytes=1887436800 egress_bytes=10479608168 ingress_bytes=12361596928 link_bytes=12361596928 total_nvlink_bytes=22841205096 weight_to_token_ratio=0.090076 prefetch_share_of_ingress=0.073289 dedup_ratio=0.064720 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=13 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.050398 gemm_inner_pad_upper_bound=11176 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.306353744e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=21656062536 prefetch_bytes=754974720 reduce_bytes=603979776 weight_movement_bytes=1358954496 egress_bytes=10830793288 ingress_bytes=12184223744 link_bytes=12184223744 total_nvlink_bytes=23015017032 weight_to_token_ratio=0.062752 prefetch_share_of_ingress=0.061963 dedup_ratio=0.044937 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=8 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.051164 gemm_inner_pad_upper_bound=10160 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.287609231e-04 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none + [2026-08-29 23:35:14] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 371.8 | throughput per GPU (TFLOP/s/GPU): 486.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.207909E+00 | loss scale: 1.0 | grad norm: 4.465 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:35:14 +[rank0]:[W829 23:35:15.585581488 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:35:15.751000 213279 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:35:15.752000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213464 closing signal SIGTERM +W0829 23:35:15.752000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213465 closing signal SIGTERM +W0829 23:35:15.753000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213466 closing signal SIGTERM +W0829 23:35:15.755000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213467 closing signal SIGTERM +W0829 23:35:15.756000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213468 closing signal SIGTERM +W0829 23:35:15.757000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213469 closing signal SIGTERM +W0829 23:35:15.757000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213470 closing signal SIGTERM +W0829 23:35:15.759000 213279 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 213471 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 213279 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..e86f261abf979093291b07daa82d84114cebb821 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/mem_samples.csv @@ -0,0 +1,232 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 3 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 +0, 629 +1, 725 +2, 725 +3, 725 +4, 725 +5, 725 +6, 725 +7, 725 +0, 963 +1, 951 +2, 983 +3, 955 +4, 975 +5, 955 +6, 987 +7, 983 +0, 1975 +1, 1975 +2, 1975 +3, 1975 +4, 1975 +5, 1975 +6, 1975 +7, 1975 +0, 17633 +1, 17537 +2, 17501 +3, 18033 +4, 17489 +5, 17585 +6, 17501 +7, 25333 +0, 48447 +1, 48449 +2, 48449 +3, 48449 +4, 48449 +5, 48449 +6, 48449 +7, 48449 +0, 56921 +1, 56913 +2, 56911 +3, 56879 +4, 56745 +5, 56969 +6, 56969 +7, 56927 +0, 58809 +1, 58829 +2, 58819 +3, 58839 +4, 58541 +5, 58841 +6, 58839 +7, 58861 +0, 59729 +1, 59729 +2, 59729 +3, 59729 +4, 59441 +5, 59731 +6, 59729 +7, 59731 +0, 59731 +1, 59731 +2, 59731 +3, 59731 +4, 59443 +5, 59731 +6, 59731 +7, 59731 +0, 85347 +1, 85389 +2, 85389 +3, 85389 +4, 85349 +5, 85389 +6, 85389 +7, 85389 +0, 87633 +1, 87675 +2, 87675 +3, 87675 +4, 87635 +5, 87675 +6, 87675 +7, 87675 +0, 87633 +1, 87675 +2, 87675 +3, 87675 +4, 87635 +5, 87675 +6, 87675 +7, 87675 +0, 88035 +1, 88077 +2, 88077 +3, 88077 +4, 88037 +5, 88077 +6, 88077 +7, 88077 +0, 96943 +1, 97055 +2, 97003 +3, 97015 +4, 96939 +5, 97057 +6, 97029 +7, 97051 +0, 120043 +1, 120081 +2, 120081 +3, 120081 +4, 120043 +5, 120081 +6, 120081 +7, 120081 +0, 120043 +1, 120081 +2, 120081 +3, 120081 +4, 120043 +5, 120081 +6, 120081 +7, 120081 +0, 120043 +1, 120081 +2, 120081 +3, 120081 +4, 120043 +5, 120081 +6, 120081 +7, 120081 +0, 118205 +1, 118243 +2, 118307 +3, 119435 +4, 118589 +5, 119435 +6, 118563 +7, 118243 +0, 117171 +1, 116821 +2, 116559 +3, 117209 +4, 116783 +5, 117209 +6, 116821 +7, 117209 +0, 65 +1, 114445 +2, 324 +3, 4 +4, 4 +5, 4 +6, 4 +7, 4 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..8a956ce8e9cde03cb82de847e20e2d03e4e15c63 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_235B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=12 +hidden=4096 +ffn_hidden=12288 +moe_ffn_hidden=1536 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=18874368 +heads=64 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=12 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=8192 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=65536 +mean_tokens_per_expert=4096 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=1 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..21a18b54fb9ac4e1331b2e3b52972390c90497fd --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/train.log @@ -0,0 +1,4277 @@ +[23:35:21] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[23:35:31] WARN: moonep_backend_async_grad_reduce_token_routing_analysis lacks --moe-router-fusion; QWEN3_235B runs unfused (timing only, routing identical) +============================================================== + QWEN3_235B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=1536 m=16 + L=12 seq=8192 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:35:32.435000 216540 torch/distributed/run.py:766] +W0829 23:35:32.435000 216540 torch/distributed/run.py:766] ***************************************** +W0829 23:35:32.435000 216540 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:35:32.435000 216540 torch/distributed/run.py:766] ***************************************** +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +using world size: 8, data-parallel size: 8, context-parallel size: 1, hierarchical context-parallel sizes: None, tensor-model-parallel size: 1, pipeline-model-parallel size: 1 +WARNING: overriding default arguments for tokenizer_type:GPT2BPETokenizer with tokenizer_type:GPT2BPETokenizer +Number of virtual stages per pipeline stage: None +accumulate and all-reduce gradients in fp32 for bfloat16 data type. +using torch.bfloat16 for parameters ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +------------------------ arguments ------------------------ + account_for_embedding_in_pipeline_split ......... False + account_for_loss_in_pipeline_split .............. False + accumulate_allreduce_grads_in_fp32 .............. True + adam_beta1 ...................................... 0.9 + adam_beta2 ...................................... 0.999 + adam_eps ........................................ 1e-08 + add_bias_linear ................................. False + add_position_embedding .......................... False + add_qkv_bias .................................... False + adlr_autoresume ................................. False + adlr_autoresume_interval ........................ 1000 + align_grad_reduce ............................... True + align_param_gather .............................. False + app_tag_run_name ................................ None + app_tag_run_version ............................. 0.0.0 + apply_layernorm_1p .............................. False + apply_query_key_layer_scaling ................... False + apply_residual_connection_post_layernorm ........ False + apply_rope_fusion ............................... True + async_save ...................................... None + async_tensor_model_parallel_allreduce ........... True + attention_backend ............................... AttnBackend.auto + attention_dropout ............................... 0.0 + attention_softmax_in_fp32 ....................... False + auto_detect_ckpt_format ......................... False + barrier_with_L1_time ............................ True + bert_binary_head ................................ True + bert_embedder_type .............................. megatron + bert_load ....................................... None + bf16 ............................................ True + bias_dropout_fusion ............................. True + bias_gelu_fusion ................................ False + bias_swiglu_fusion .............................. True + biencoder_projection_dim ........................ 0 + biencoder_shared_query_context_model ............ False + block_data_path ................................. None + calc_ft_timeouts ................................ False + calculate_per_token_loss ........................ False + check_for_large_grads ........................... False + check_for_nan_in_loss_and_grad .................. True + check_for_spiky_loss ............................ False + check_weight_hash_across_dp_replicas_interval ... None + ckpt_assume_constant_structure .................. False + ckpt_convert_format ............................. None + ckpt_convert_save ............................... None + ckpt_convert_update_legacy_dist_opt_format ...... False + ckpt_format ..................................... torch_dist + ckpt_fully_parallel_load ........................ False + ckpt_fully_parallel_save ........................ True + ckpt_fully_parallel_save_deprecated ............. False + ckpt_step ....................................... None + classes_fraction ................................ 1.0 + clip_grad ....................................... 1.0 + clone_scatter_output_in_embedding ............... True + config_logger_dir ............................... + consumed_train_samples .......................... 0 + consumed_valid_samples .......................... 0 + context_parallel_size ........................... 1 + cp_comm_type .................................... ['p2p'] + create_attention_mask_in_dataloader ............. True + cross_entropy_fusion_impl ....................... native + cross_entropy_loss_fusion ....................... False + cuda_graph_scope ................................ full + cuda_graph_warmup_steps ......................... 3 + data_args_path .................................. None + data_cache_path ................................. None + data_parallel_random_init ....................... False + data_parallel_sharding_strategy ................. no_shard + data_parallel_size .............................. 8 + data_path ....................................... ['/workspace/data/wikitext/wikitext103_gpt2_text_document'] + data_per_class_fraction ......................... 1.0 + data_sharding ................................... True + dataloader_type ................................. single + ddp_average_in_collective ....................... False + ddp_bucket_size ................................. None + ddp_num_buckets ................................. None + ddp_pad_buckets_for_high_nccl_busbw ............. False + decoder_first_pipeline_num_layers ............... None + decoder_last_pipeline_num_layers ................ None + decoder_num_layers .............................. None + decoder_seq_length .............................. None + decoupled_lr .................................... None + decoupled_min_lr ................................ None + decrease_batch_size_if_needed ................... False + defer_embedding_wgrad_compute ................... False + delay_wgrad_compute ............................. False + deprecated_use_mcore_models ..................... True + deterministic_mode .............................. False + dino_bottleneck_size ............................ 256 + dino_freeze_last_layer .......................... 1 + dino_head_hidden_size ........................... 2048 + dino_local_crops_number ......................... 10 + dino_local_img_size ............................. 96 + dino_norm_last_layer ............................ False + dino_teacher_temp ............................... 0.07 + dino_warmup_teacher_temp ........................ 0.04 + dino_warmup_teacher_temp_epochs ................. 30 + disable_bf16_reduced_precision_matmul ........... False + disable_mamba_mem_eff_path ...................... False + disable_straggler_on_startup .................... False + dist_ckpt_format_deprecated ..................... None + dist_ckpt_strictness ............................ assume_ok_unexpected + distribute_saved_activations .................... False + distributed_backend ............................. nccl + distributed_timeout_minutes ..................... 10 + embedding_init_method_std ....................... None + embedding_path .................................. None + empty_unused_memory_level ....................... 0 + enable_cuda_graph ............................... False + enable_experimental ............................. False + enable_ft_package ............................... False + enable_gloo_process_groups ...................... True + enable_msc ...................................... True + enable_one_logger ............................... True + encoder_num_layers .............................. 12 + encoder_seq_length .............................. 8192 + end_weight_decay ................................ 0.1 + eod_mask_loss ................................... False + error_injection_rate ............................ 0 + error_injection_type ............................ transient_error + eval_interval ................................... 1000000 + eval_iters ...................................... 0 + evidence_data_path .............................. None + exit_duration_in_mins ........................... None + exit_interval ................................... None + exit_on_missing_checkpoint ...................... False + exit_signal_handler ............................. False + exp_avg_dtype ................................... torch.float32 + exp_avg_sq_dtype ................................ torch.float32 + expert_model_parallel_size ...................... 8 + expert_tensor_parallel_size ..................... 1 + export_force_local_attention .................... False + export_kd_cfg ................................... None + export_kd_teacher_ckpt_format ................... None + export_kd_teacher_load .......................... None + export_kv_cache_quant ........................... False + export_legacy_megatron .......................... False + export_model_type ............................... GPTModel + export_moe_apply_probs_on_input ................. False + export_qk_l2_norm ............................... False + export_quant_cfg ................................ None + export_real_quant_cfg ........................... None + export_te_mcore_model ........................... False + external_cuda_graph ............................. False + ffn_hidden_size ................................. 12288 + finetune ........................................ False + finetune_data_split ............................. train + finetune_hf_dataset ............................. None + first_last_layers_bf16 .......................... False + flash_decode .................................... False + fp16 ............................................ False + fp16_lm_cross_entropy ........................... False + fp32_residual_connection ........................ False + fp8 ............................................. e4m3 + fp8_amax_compute_algo ........................... most_recent + fp8_amax_history_len ............................ 1 + fp8_interval .................................... 1 + fp8_margin ...................................... 0 + fp8_param_gather ................................ False + fp8_recipe ...................................... mxfp8 + fp8_wgrad ....................................... True + fsdp_double_buffer .............................. False + global_batch_size ............................... 8 + grad_reduce_in_bf16 ............................. False + gradient_accumulation_fusion .................... True + gradient_reduce_div_fusion ...................... True + group_query_attention ........................... True + head_lr_mult .................................... 1.0 + heterogeneous_layers_config_encoded_json ........ None + heterogeneous_layers_config_path ................ None + hidden_dropout .................................. 0.0 + hidden_size ..................................... 4096 + hierarchical_context_parallel_sizes ............. None + high_priority_stream_groups ..................... [] + hybrid_attention_ratio .......................... 0.0 + hybrid_mlp_ratio ................................ 0.0 + hybrid_override_pattern ......................... None + hysteresis ...................................... 2 + ict_head_size ................................... None + ict_load ........................................ None + img_h ........................................... 224 + img_w ........................................... 224 + indexer_batch_size .............................. 128 + indexer_log_interval ............................ 1000 + inference_batch_times_seqlen_threshold .......... -1 + inference_dynamic_batching ...................... False + inference_dynamic_batching_buffer_guaranteed_fraction 0.2 + inference_dynamic_batching_buffer_overflow_factor None + inference_dynamic_batching_buffer_size_gb ....... 40.0 + inference_dynamic_batching_chunk_size ........... 256 + inference_dynamic_batching_max_requests_override None + inference_dynamic_batching_max_tokens_override .. None + inference_dynamic_batching_num_cuda_graphs ...... 16 + inference_max_batch_size ........................ 8 + inference_max_seq_length ........................ 2560 + inference_rng_tracker ........................... False + init_method_std ................................. 0.01 + init_method_xavier_uniform ...................... False + init_model_with_meta_device ..................... False + initial_loss_scale .............................. 4294967296 + inprocess_active_world_size ..................... 8 + inprocess_barrier_timeout ....................... 120 + inprocess_completion_timeout .................... 120 + inprocess_empty_cuda_cache ...................... False + inprocess_granularity ........................... node + inprocess_hard_timeout .......................... 90 + inprocess_heartbeat_interval .................... 30 + inprocess_heartbeat_timeout ..................... 60 + inprocess_last_call_wait ........................ 1 + inprocess_max_iterations ........................ None + inprocess_monitor_process_interval .............. 1.0 + inprocess_monitor_thread_interval ............... 1.0 + inprocess_progress_watchdog_interval ............ 1.0 + inprocess_restart ............................... False + inprocess_soft_timeout .......................... 60 + inprocess_termination_grace_time ................ 1 + is_hybrid_model ................................. False + iter_per_epoch .................................. 1250 + iterations_to_skip .............................. [] + keep_fp8_transpose_cache_when_using_custom_fsdp . False + kitchen_config_file ............................. None + kitchen_recipe_number ........................... None + kv_channels ..................................... 128 + kv_lora_rank .................................... 32 + lazy_mpu_init ................................... None + load ............................................ None + load_main_params_from_ckpt ...................... None + load_model_opt_format ........................... False + local_rank ...................................... 0 + log_energy ...................................... False + log_interval .................................... 1 + log_loss_scale_to_tensorboard ................... True + log_memory_to_tensorboard ....................... False + log_num_zeros_in_grad ........................... False + log_params_norm ................................. False + log_progress .................................... False + log_straggler ................................... False + log_throughput .................................. True + log_timers_to_tensorboard ....................... False + log_validation_ppl_to_tensorboard ............... False + log_world_size_to_tensorboard ................... False + logging_level ................................... None + loss_scale ...................................... None + loss_scale_window ............................... 1000 + lr .............................................. 0.0001 + lr_decay_iters .................................. 8 + lr_decay_samples ................................ None + lr_decay_style .................................. cosine + lr_warmup_fraction .............................. None + lr_warmup_init .................................. 0.0 + lr_warmup_iters ................................. 5 + lr_warmup_samples ............................... 0 + lr_wsd_decay_iters .............................. None + lr_wsd_decay_samples ............................ None + lr_wsd_decay_style .............................. exponential + main_grads_dtype ................................ torch.float32 + main_params_dtype ............................... torch.float32 + make_vocab_size_divisible_by .................... 128 + mamba_head_dim .................................. 64 + mamba_num_groups ................................ 8 + mamba_num_heads ................................. None + mamba_state_dim ................................. 128 + manual_gc ....................................... False + manual_gc_eval .................................. True + manual_gc_interval .............................. 0 + mask_factor ..................................... 1.0 + mask_prob ....................................... 0.15 + mask_type ....................................... random + masked_softmax_fusion ........................... False + max_position_embeddings ......................... 32768 + max_tokens_to_oom ............................... 12000 + memory_snapshot_path ............................ snapshot.pickle + merge_file ...................................... /workspace/data/wikitext/gpt2-merges.txt + micro_batch_size ................................ 1 + microbatch_group_size_per_vp_stage .............. None + mid_level_dataset_surplus ....................... 0.005 + min_loss_scale .................................. 1.0 + min_lr .......................................... 1e-05 + mlp_chunks_for_prefill .......................... 1 + mmap_bin_files .................................. True + mock_data ....................................... False + moe_apply_probs_on_input ........................ False + moe_aux_loss_coeff .............................. 0.001 + moe_deepep_num_sms .............................. 20 + moe_echo_enable_random_offloading ............... False + moe_echo_expert_dispatch_overlap ................ False + moe_echo_expert_dispatcher_type ................. hybridep + moe_echo_recompute_expert_dispatch .............. False + moe_enable_deepep ............................... False + moe_enable_echo ................................. False + moe_enable_hybridep ............................. False + moe_enable_moonep ............................... True + moe_expert_capacity_factor ...................... None + moe_extended_tp ................................. False + moe_ffn_hidden_size ............................. 1536 + moe_grouped_gemm ................................ True + moe_hybridep_num_sms ............................ 16 + moe_input_jitter_eps ............................ None + moe_layer_freq .................................. 1 + moe_layer_recompute ............................. False + moe_moonep_async_grad_reduce .................... True + moe_moonep_num_sms .............................. None + moe_moonep_traffic_logging ...................... True + moe_moonep_traffic_logging_interval ............. 1 + moe_moonep_traffic_warmup_iters ................. 2 + moe_num_echo_experts ............................ None + moe_pad_expert_input_to_capacity ................ False + moe_per_layer_logging ........................... False + moe_permute_fusion .............................. False + moe_received_token_capacity ..................... None + moe_router_bias_update_rate ..................... 0.001 + moe_router_dtype ................................ fp32 + moe_router_enable_expert_bias ................... False + moe_router_force_load_balancing ................. True + moe_router_group_topk ........................... None + moe_router_load_balancing_type .................. aux_loss + moe_router_num_groups ........................... None + moe_router_padding_for_fp8 ...................... False + moe_router_pre_softmax .......................... False + moe_router_score_function ....................... softmax + moe_router_topk ................................. 8 + moe_router_topk_scaling_factor .................. None + moe_shared_expert_intermediate_size ............. None + moe_shared_expert_overlap ....................... False + moe_token_dispatcher_type ....................... flex + moe_token_drop_policy ........................... probs + moe_upcycling_granularity ....................... 1 + moe_use_device_initiated_grouped_gemm ........... True + moe_use_legacy_grouped_gemm ..................... False + moe_use_upcycling ............................... False + moe_z_loss_coeff ................................ None + mrope_section ................................... None + mscale .......................................... 1.0 + mscale_all_dim .................................. 0.0 + mtp_loss_scaling_factor ......................... 0.1 + mtp_num_layers .................................. None + multi_latent_attention .......................... False + nccl_all_reduce_for_prefill ..................... False + nccl_communicator_config_path ................... None + nccl_ub ......................................... False + no_load_optim ................................... None + no_load_rng ..................................... None + no_persist_layer_norm ........................... False + no_rope_freq .................................... None + no_save_optim ................................... None + no_save_rng ..................................... None + non_persistent_ckpt_type ........................ None + non_persistent_global_ckpt_dir .................. None + non_persistent_local_ckpt_algo .................. fully_parallel + non_persistent_local_ckpt_dir ................... None + non_persistent_save_interval .................... None + norm_epsilon .................................... 1e-06 + normalization ................................... RMSNorm + num_attention_heads ............................. 64 + num_channels .................................... 3 + num_classes ..................................... 1000 + num_dataset_builder_threads ..................... 1 + num_distributed_optimizer_instances ............. 1 + num_experts ..................................... 128 + num_layers ...................................... 12 + num_layers_at_end_in_bf16 ....................... 1 + num_layers_at_start_in_bf16 ..................... 1 + num_layers_per_virtual_pipeline_stage ........... None + num_query_groups ................................ 4 + num_virtual_stages_per_pipeline_rank ............ None + num_workers ..................................... 2 + object_storage_cache_path ....................... None + one_logger_async ................................ False + one_logger_project .............................. megatron-lm + one_logger_run_name ............................. None + onnx_safe ....................................... None + openai_gelu ..................................... False + optimizer ....................................... adam + optimizer_cpu_offload ........................... False + optimizer_offload_fraction ...................... 1.0 + output_bert_embeddings .......................... False + overlap_cpu_optimizer_d2h_h2d ................... False + overlap_grad_reduce ............................. True + overlap_moe_expert_parallel_comm ................ False + overlap_p2p_comm ................................ False + overlap_p2p_comm_warmup_flush ................... False + overlap_param_gather ............................ True + overlap_param_gather_with_optimizer_step ........ False + override_opt_param_scheduler .................... False + params_dtype .................................... torch.bfloat16 + patch_dim ....................................... 16 + per_split_data_args_path ........................ None + perform_initialization .......................... True + pin_cpu_grads ................................... True + pin_cpu_params .................................. True + pipeline_model_parallel_comm_backend ............ None + pipeline_model_parallel_layout .................. None + pipeline_model_parallel_size .................... 1 + position_embedding_type ......................... rope + pretrained_checkpoint ........................... None + profile ......................................... False + profile_ranks ................................... [0] + profile_step_end ................................ 12 + profile_step_start .............................. 10 + q_lora_rank ..................................... None + qk_head_dim ..................................... 128 + qk_l2_norm ...................................... False + qk_layernorm .................................... True + qk_pos_emb_head_dim ............................. 64 + query_in_block_prob ............................. 0.1 + rampup_batch_size ............................... None + rank ............................................ 0 + recompute_granularity ........................... None + recompute_method ................................ None + recompute_modules ............................... None + recompute_num_layers ............................ None + record_memory_history ........................... False + relative_attention_max_distance ................. 128 + relative_attention_num_buckets .................. 32 + replication ..................................... False + replication_factor .............................. 2 + replication_jump ................................ None + rerun_mode ...................................... validate_results + reset_attention_mask ............................ False + reset_position_ids .............................. False + result_rejected_tracker_filename ................ None + retriever_report_topk_accuracies ................ [] + retriever_score_scaling ......................... False + retriever_seq_length ............................ 256 + retro_add_retriever ............................. False + retro_attention_gate ............................ 1 + retro_cyclic_train_iters ........................ None + retro_encoder_attention_dropout ................. 0.1 + retro_encoder_hidden_dropout .................... 0.1 + retro_encoder_layers ............................ 2 + retro_num_neighbors ............................. 2 + retro_num_retrieved_chunks ...................... 2 + retro_project_dir ............................... None + retro_verify_neighbor_count ..................... True + reuse_grad_buf_for_mxfp8_param_ag ............... False + rope_scaling_factor ............................. 8.0 + rope_type ....................................... None + rotary_base ..................................... 1000000 + rotary_interleaved .............................. False + rotary_percent .................................. 1.0 + rotary_scaling_factor ........................... 1.0 + rotary_seq_len_interpolation_factor ............. None + run_workload_inspector_server ................... False + sample_rate ..................................... 1.0 + save ............................................ None + save_interval ................................... None + save_retain_interval ............................ None + scatter_gather_tensors_in_pipeline .............. True + seed ............................................ 1234 + seq_length ...................................... 8192 + sequence_parallel ............................... False + sft ............................................. False + sft_tokenizer_prompt_format ..................... nemotron-h-aligned + sgd_momentum .................................... 0.9 + short_seq_prob .................................. 0.1 + skip_train ...................................... False + skipped_train_samples ........................... 0 + spec ............................................ None + split ........................................... 100,0,0 + squared_relu .................................... False + start_weight_decay .............................. 0.1 + straggler_ctrlr_port ............................ 65535 + straggler_minmax_count .......................... 1 + suggested_communication_unit_size ............... None + swiglu .......................................... True + swin_backbone_type .............................. tiny + symmetric_ar_type ............................... None + te_rng_tracker .................................. False + tensor_model_parallel_size ...................... 1 + tensorboard_dir ................................. None + tensorboard_log_interval ........................ 1 + tensorboard_queue_size .......................... 1000 + test_data_path .................................. None + test_mode ....................................... False + tiktoken_num_special_tokens ..................... 1000 + tiktoken_pattern ................................ None + tiktoken_special_tokens ......................... None + timing_log_level ................................ 0 + timing_log_option ............................... minmax + titles_data_path ................................ None + tokenizer_model ................................. None + tokenizer_type .................................. GPT2BPETokenizer + torch_fsdp2_reshard_after_forward ............... True + tp_comm_bootstrap_backend ....................... nccl + tp_comm_bulk_dgrad .............................. True + tp_comm_bulk_wgrad .............................. True + tp_comm_overlap ................................. False + tp_comm_overlap_ag .............................. True + tp_comm_overlap_cfg ............................. None + tp_comm_overlap_rs .............................. True + tp_comm_overlap_rs_dgrad ........................ False + tp_comm_split_ag ................................ True + tp_comm_split_rs ................................ True + train_data_path ................................. None + train_iters ..................................... 12 + train_samples ................................... None + train_sync_interval ............................. None + transformer_impl ................................ transformer_engine + transformer_pipeline_model_parallel_size ........ 1 + untie_embeddings_and_output_weights ............. True + use_checkpoint_args ............................. False + use_checkpoint_opt_param_scheduler .............. False + use_cpu_initialization .......................... None + use_custom_fsdp ................................. False + use_dist_ckpt ................................... True + use_dist_ckpt_deprecated ........................ False + use_distributed_optimizer ....................... True + use_flash_attn .................................. False + use_legacy_models ............................... False + use_mp_args_from_checkpoint_args ................ False + use_one_sent_docs ............................... False + use_persistent_ckpt_worker ...................... False + use_precision_aware_optimizer ................... False + use_pytorch_profiler ............................ False + use_ring_exchange_p2p ........................... False + use_rope_scaling ................................ False + use_rotary_position_embeddings .................. False + use_sharp ....................................... False + use_tokenizer_model_from_checkpoint_args ........ True + use_torch_fsdp2 ................................. False + use_torch_optimizer_for_cpu_offload ............. False + use_tp_pp_dp_mapping ............................ False + v_head_dim ...................................... 128 + valid_data_path ................................. None + variable_seq_lengths ............................ False + virtual_pipeline_model_parallel_size ............ None + vision_backbone_type ............................ vit + vision_pretraining .............................. False + vision_pretraining_type ......................... classify + vocab_extra_ids ................................. 0 + vocab_file ...................................... /workspace/data/wikitext/gpt2-vocab.json + vocab_size ...................................... None + wandb_exp_name .................................. + wandb_project ................................... + wandb_save_dir .................................. + weight_decay .................................... 0.1 + weight_decay_incr_style ......................... constant + wgrad_deferral_limit ............................ 0 + world_size ...................................... 8 + yaml_cfg ........................................ None +-------------------- end of arguments --------------------- +INFO:megatron.core.num_microbatches_calculator:setting number of microbatches to constant 1 +> building GPT2BPETokenizer tokenizer ... + > padded vocab (size: 50257) with 47 dummy tokens (new size: 50304) +WARNING:megatron.core.rerun_state_machine:RerunStateMachine initialized in mode RerunMode.VALIDATE_RESULTS +> initializing torch distributed ... +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +/workspace/Megatron-MoE-Echo/megatron/training/arguments.py:831: UserWarning: Disabling sequence parallelism because tensor model parallelism is disabled + warnings.warn("Disabling sequence parallelism because tensor model parallelism is disabled") +WARNING: one_logger package is required to enable e2e metrics tracking. please go to https://confluence.nvidia.com/display/MLWFO/Package+Repositories for details to install it +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 3 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 2 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 4 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 5 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 7 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7[Gloo] Rank +6 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 1 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank 0 is connected to 7 peer ranks. Expected number of connected peer ranks is : 7 +[Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank [Gloo] Rank 3[Gloo] Rank 4762 is connected to 5 is connected to is connected to is connected to is connected to 7 is connected to 7777 peer ranks. 7 peer ranks. peer ranks. peer ranks. peer ranks. Expected number of connected peer ranks is : peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : Expected number of connected peer ranks is : 7Expected number of connected peer ranks is : 7777 +7 + + + + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +> initialized tensor model parallel with size 1 +> initialized pipeline model parallel with size 1 +> setting random seeds to 1234 ... +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank [Gloo] Rank 00 is connected to is connected to 00 peer ranks. peer ranks. Expected number of connected peer ranks is : Expected number of connected peer ranks is : 00 + +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[Gloo] Rank 0 is connected to 0 peer ranks. Expected number of connected peer ranks is : 0 +[rank6]:[W829 23:35:51.256579825 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 6] using GPU 6 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank1]:[W829 23:35:51.256691165 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 1] using GPU 1 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank2]:[W829 23:35:51.256741115 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 2] using GPU 2 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank5]:[W829 23:35:51.256747782 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 5] using GPU 5 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank7]:[W829 23:35:51.256790656 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 7] using GPU 7 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank3]:[W829 23:35:51.256799226 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 3] using GPU 3 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +[rank4]:[W829 23:35:51.257868691 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 4] using GPU 4 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +> compiling dataset index builder ... +make: Entering directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +make: Nothing to be done for 'default'. +make: Leaving directory '/workspace/Megatron-MoE-Echo/megatron/core/datasets' +>>> done with dataset index builder. Compilation time: 0.033 seconds +WARNING: constraints for invoking optimized fused softmax kernel are not met. We default back to unfused kernel invocations. +> compiling and loading fused kernels ... +[rank0]:[W829 23:35:51.460664958 ProcessGroupNCCL.cpp:4789] [PG ID 0 PG GUID 0 Rank 0] using GPU 0 as device used by this process is currently unknown. This can potentially cause a hang if this rank to GPU mapping is incorrect. You can specify device_id in init_process_group() to force use of a particular device. +>>> done with compiling and loading fused kernels. Compilation time: 3.868 seconds +time to initialize megatron (seconds): 8.019 +[after megatron is initialized] datetime: 2026-08-29 23:35:57 +building GPT model ... +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/transformer_engine/pytorch/cpu_offload.py:708: DeprecationWarning: Offloading weights is deprecated. Using offload_weights=True does not have any effect. + warnings.warn( + > number of parameters on (tensor, pipeline) model parallel rank (0, 0): 4898003968 +INFO:megatron.core.distributed.distributed_data_parallel:Setting up DistributedDataParallel with config DistributedDataParallelConfig(grad_reduce_in_fp32=True, overlap_grad_reduce=True, overlap_param_gather=True, align_param_gather=False, use_distributed_optimizer=True, num_distributed_optimizer_instances=1, check_for_nan_in_grad=True, check_for_large_grads=False, bucket_size=40000000, pad_buckets_for_high_nccl_busbw=False, average_in_collective=False, fp8_param_gather=False, reuse_grad_buf_for_mxfp8_param_ag=False, use_custom_fsdp=False, data_parallel_sharding_strategy='no_shard', gradient_reduce_div_fusion=True, suggested_communication_unit_size=None, preserve_fp32_weights=True, keep_fp8_transpose_cache_when_using_custom_fsdp=False, nccl_ub=False, fsdp_double_buffer=False) +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 14 +Params for bucket 1 (206045184 elements, 206045184 padded size): + module.output_layer.weight +Params for bucket 2 (71840000 elements, 71840000 padded size): + module.decoder.layers.11.pre_mlp_layernorm.weight + module.decoder.layers.11.self_attention.k_layernorm.weight + module.decoder.layers.11.mlp.router.weight + module.decoder.layers.11.self_attention.q_layernorm.weight + module.decoder.layers.11.self_attention.linear_qkv.weight + module.decoder.layers.11.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.11.self_attention.linear_proj.weight + module.decoder.final_layernorm.weight +Params for bucket 3 (71835904 elements, 71835904 padded size): + module.decoder.layers.10.mlp.router.weight + module.decoder.layers.10.self_attention.linear_qkv.weight + module.decoder.layers.10.pre_mlp_layernorm.weight + module.decoder.layers.10.self_attention.k_layernorm.weight + module.decoder.layers.10.self_attention.q_layernorm.weight + module.decoder.layers.10.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.10.self_attention.linear_proj.weight +Params for bucket 4 (71835904 elements, 71835904 padded size): + module.decoder.layers.9.mlp.router.weight + module.decoder.layers.9.self_attention.k_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.weight + module.decoder.layers.9.self_attention.q_layernorm.weight + module.decoder.layers.9.pre_mlp_layernorm.weight + module.decoder.layers.9.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.9.self_attention.linear_proj.weight +Params for bucket 5 (71835904 elements, 71835904 padded size): + module.decoder.layers.8.mlp.router.weight + module.decoder.layers.8.self_attention.linear_qkv.weight + module.decoder.layers.8.self_attention.q_layernorm.weight + module.decoder.layers.8.pre_mlp_layernorm.weight + module.decoder.layers.8.self_attention.k_layernorm.weight + module.decoder.layers.8.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.8.self_attention.linear_proj.weight +Params for bucket 6 (71835904 elements, 71835904 padded size): + module.decoder.layers.7.mlp.router.weight + module.decoder.layers.7.self_attention.q_layernorm.weight + module.decoder.layers.7.self_attention.k_layernorm.weight + module.decoder.layers.7.pre_mlp_layernorm.weight + module.decoder.layers.7.self_attention.linear_qkv.weight + module.decoder.layers.7.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.7.self_attention.linear_proj.weight +Params for bucket 7 (71835904 elements, 71835904 padded size): + module.decoder.layers.6.mlp.router.weight + module.decoder.layers.6.pre_mlp_layernorm.weight + module.decoder.layers.6.self_attention.linear_qkv.weight + module.decoder.layers.6.self_attention.q_layernorm.weight + module.decoder.layers.6.self_attention.k_layernorm.weight + module.decoder.layers.6.self_attention.linear_proj.weight + module.decoder.layers.6.self_attention.linear_qkv.layer_norm_weight +Params for bucket 8 (71835904 elements, 71835904 padded size): + module.decoder.layers.5.self_attention.q_layernorm.weight + module.decoder.layers.5.mlp.router.weight + module.decoder.layers.5.pre_mlp_layernorm.weight + module.decoder.layers.5.self_attention.k_layernorm.weight + module.decoder.layers.5.self_attention.linear_qkv.weight + module.decoder.layers.5.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.5.self_attention.linear_proj.weight +Params for bucket 9 (71835904 elements, 71835904 padded size): + module.decoder.layers.4.mlp.router.weight + module.decoder.layers.4.self_attention.q_layernorm.weight + module.decoder.layers.4.self_attention.linear_proj.weight + module.decoder.layers.4.pre_mlp_layernorm.weight + module.decoder.layers.4.self_attention.k_layernorm.weight + module.decoder.layers.4.self_attention.linear_qkv.weight + module.decoder.layers.4.self_attention.linear_qkv.layer_norm_weight +Params for bucket 10 (71835904 elements, 71835904 padded size): + module.decoder.layers.3.mlp.router.weight + module.decoder.layers.3.self_attention.k_layernorm.weight + module.decoder.layers.3.self_attention.q_layernorm.weight + module.decoder.layers.3.pre_mlp_layernorm.weight + module.decoder.layers.3.self_attention.linear_qkv.weight + module.decoder.layers.3.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.3.self_attention.linear_proj.weight +Params for bucket 11 (71835904 elements, 71835904 padded size): + module.decoder.layers.2.mlp.router.weight + module.decoder.layers.2.self_attention.k_layernorm.weight + module.decoder.layers.2.self_attention.q_layernorm.weight + module.decoder.layers.2.pre_mlp_layernorm.weight + module.decoder.layers.2.self_attention.linear_qkv.weight + module.decoder.layers.2.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.2.self_attention.linear_proj.weight +Params for bucket 12 (71835904 elements, 71835904 padded size): + module.decoder.layers.1.mlp.router.weight + module.decoder.layers.1.pre_mlp_layernorm.weight + module.decoder.layers.1.self_attention.k_layernorm.weight + module.decoder.layers.1.self_attention.q_layernorm.weight + module.decoder.layers.1.self_attention.linear_qkv.weight + module.decoder.layers.1.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.1.self_attention.linear_proj.weight +Params for bucket 13 (71835904 elements, 71835904 padded size): + module.decoder.layers.0.self_attention.linear_qkv.weight + module.decoder.layers.0.self_attention.q_layernorm.weight + module.decoder.layers.0.pre_mlp_layernorm.weight + module.decoder.layers.0.self_attention.linear_qkv.layer_norm_weight + module.decoder.layers.0.mlp.router.weight + module.decoder.layers.0.self_attention.k_layernorm.weight + module.decoder.layers.0.self_attention.linear_proj.weight +Params for bucket 14 (206045184 elements, 206045184 padded size): + module.embedding.word_embeddings.weight +INFO:megatron.core.distributed.param_and_grad_buffer:Number of buckets for gradient all-reduce / reduce-scatter: 76 +Params for bucket 1 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight3 + module.decoder.layers.11.mlp.experts.linear_fc2.weight2 + module.decoder.layers.11.mlp.experts.linear_fc2.weight5 + module.decoder.layers.11.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc2.weight1 + module.decoder.layers.11.mlp.experts.linear_fc2.weight6 +Params for bucket 2 (44040192 elements, 44040192 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight7 + module.decoder.layers.11.mlp.experts.linear_fc2.weight11 + module.decoder.layers.11.mlp.experts.linear_fc2.weight9 + module.decoder.layers.11.mlp.experts.linear_fc2.weight13 + module.decoder.layers.11.mlp.experts.linear_fc2.weight10 + module.decoder.layers.11.mlp.experts.linear_fc2.weight8 + module.decoder.layers.11.mlp.experts.linear_fc2.weight12 +Params for bucket 3 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc2.weight15 + module.decoder.layers.11.mlp.experts.linear_fc1.weight0 + module.decoder.layers.11.mlp.experts.linear_fc2.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight2 + module.decoder.layers.11.mlp.experts.linear_fc1.weight1 +Params for bucket 4 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight3 + module.decoder.layers.11.mlp.experts.linear_fc1.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight5 + module.decoder.layers.11.mlp.experts.linear_fc1.weight6 +Params for bucket 5 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight7 + module.decoder.layers.11.mlp.experts.linear_fc1.weight8 + module.decoder.layers.11.mlp.experts.linear_fc1.weight10 + module.decoder.layers.11.mlp.experts.linear_fc1.weight9 +Params for bucket 6 (50331648 elements, 50331648 padded size): + module.decoder.layers.11.mlp.experts.linear_fc1.weight11 + module.decoder.layers.11.mlp.experts.linear_fc1.weight14 + module.decoder.layers.11.mlp.experts.linear_fc1.weight12 + module.decoder.layers.11.mlp.experts.linear_fc1.weight13 +Params for bucket 7 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight1 + module.decoder.layers.10.mlp.experts.linear_fc2.weight4 + module.decoder.layers.11.mlp.experts.linear_fc1.weight15 + module.decoder.layers.10.mlp.experts.linear_fc2.weight3 + module.decoder.layers.10.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc2.weight0 +Params for bucket 8 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight6 + module.decoder.layers.10.mlp.experts.linear_fc2.weight8 + module.decoder.layers.10.mlp.experts.linear_fc2.weight10 + module.decoder.layers.10.mlp.experts.linear_fc2.weight5 + module.decoder.layers.10.mlp.experts.linear_fc2.weight9 + module.decoder.layers.10.mlp.experts.linear_fc2.weight7 + module.decoder.layers.10.mlp.experts.linear_fc2.weight11 +Params for bucket 9 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc2.weight12 + module.decoder.layers.10.mlp.experts.linear_fc2.weight14 + module.decoder.layers.10.mlp.experts.linear_fc1.weight1 + module.decoder.layers.10.mlp.experts.linear_fc1.weight0 + module.decoder.layers.10.mlp.experts.linear_fc2.weight13 + module.decoder.layers.10.mlp.experts.linear_fc2.weight15 +Params for bucket 10 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight3 + module.decoder.layers.10.mlp.experts.linear_fc1.weight4 + module.decoder.layers.10.mlp.experts.linear_fc1.weight5 +Params for bucket 11 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight6 + module.decoder.layers.10.mlp.experts.linear_fc1.weight9 + module.decoder.layers.10.mlp.experts.linear_fc1.weight7 + module.decoder.layers.10.mlp.experts.linear_fc1.weight8 +Params for bucket 12 (50331648 elements, 50331648 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight10 + module.decoder.layers.10.mlp.experts.linear_fc1.weight12 + module.decoder.layers.10.mlp.experts.linear_fc1.weight13 + module.decoder.layers.10.mlp.experts.linear_fc1.weight11 +Params for bucket 13 (44040192 elements, 44040192 padded size): + module.decoder.layers.10.mlp.experts.linear_fc1.weight15 + module.decoder.layers.9.mlp.experts.linear_fc2.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight2 + module.decoder.layers.10.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight1 +Params for bucket 14 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight3 + module.decoder.layers.9.mlp.experts.linear_fc2.weight7 + module.decoder.layers.9.mlp.experts.linear_fc2.weight5 + module.decoder.layers.9.mlp.experts.linear_fc2.weight9 + module.decoder.layers.9.mlp.experts.linear_fc2.weight4 + module.decoder.layers.9.mlp.experts.linear_fc2.weight6 + module.decoder.layers.9.mlp.experts.linear_fc2.weight8 +Params for bucket 15 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc2.weight13 + module.decoder.layers.9.mlp.experts.linear_fc2.weight11 + module.decoder.layers.9.mlp.experts.linear_fc2.weight15 + module.decoder.layers.9.mlp.experts.linear_fc1.weight0 + module.decoder.layers.9.mlp.experts.linear_fc2.weight10 + module.decoder.layers.9.mlp.experts.linear_fc2.weight14 + module.decoder.layers.9.mlp.experts.linear_fc2.weight12 +Params for bucket 16 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight1 + module.decoder.layers.9.mlp.experts.linear_fc1.weight2 + module.decoder.layers.9.mlp.experts.linear_fc1.weight4 + module.decoder.layers.9.mlp.experts.linear_fc1.weight3 +Params for bucket 17 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight5 + module.decoder.layers.9.mlp.experts.linear_fc1.weight7 + module.decoder.layers.9.mlp.experts.linear_fc1.weight6 + module.decoder.layers.9.mlp.experts.linear_fc1.weight8 +Params for bucket 18 (50331648 elements, 50331648 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight9 + module.decoder.layers.9.mlp.experts.linear_fc1.weight10 + module.decoder.layers.9.mlp.experts.linear_fc1.weight11 + module.decoder.layers.9.mlp.experts.linear_fc1.weight12 +Params for bucket 19 (44040192 elements, 44040192 padded size): + module.decoder.layers.9.mlp.experts.linear_fc1.weight13 + module.decoder.layers.9.mlp.experts.linear_fc1.weight14 + module.decoder.layers.9.mlp.experts.linear_fc1.weight15 + module.decoder.layers.8.mlp.experts.linear_fc2.weight0 +Params for bucket 20 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight1 + module.decoder.layers.8.mlp.experts.linear_fc2.weight4 + module.decoder.layers.8.mlp.experts.linear_fc2.weight6 + module.decoder.layers.8.mlp.experts.linear_fc2.weight5 + module.decoder.layers.8.mlp.experts.linear_fc2.weight2 + module.decoder.layers.8.mlp.experts.linear_fc2.weight3 + module.decoder.layers.8.mlp.experts.linear_fc2.weight7 +Params for bucket 21 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight8 + module.decoder.layers.8.mlp.experts.linear_fc2.weight12 + module.decoder.layers.8.mlp.experts.linear_fc2.weight10 + module.decoder.layers.8.mlp.experts.linear_fc2.weight14 + module.decoder.layers.8.mlp.experts.linear_fc2.weight11 + module.decoder.layers.8.mlp.experts.linear_fc2.weight9 + module.decoder.layers.8.mlp.experts.linear_fc2.weight13 +Params for bucket 22 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc2.weight15 + module.decoder.layers.8.mlp.experts.linear_fc1.weight2 + module.decoder.layers.8.mlp.experts.linear_fc1.weight1 + module.decoder.layers.8.mlp.experts.linear_fc1.weight0 +Params for bucket 23 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight3 + module.decoder.layers.8.mlp.experts.linear_fc1.weight5 + module.decoder.layers.8.mlp.experts.linear_fc1.weight6 + module.decoder.layers.8.mlp.experts.linear_fc1.weight4 +Params for bucket 24 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight7 + module.decoder.layers.8.mlp.experts.linear_fc1.weight8 + module.decoder.layers.8.mlp.experts.linear_fc1.weight9 + module.decoder.layers.8.mlp.experts.linear_fc1.weight10 +Params for bucket 25 (50331648 elements, 50331648 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight11 + module.decoder.layers.8.mlp.experts.linear_fc1.weight14 + module.decoder.layers.8.mlp.experts.linear_fc1.weight12 + module.decoder.layers.8.mlp.experts.linear_fc1.weight13 +Params for bucket 26 (44040192 elements, 44040192 padded size): + module.decoder.layers.8.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight4 + module.decoder.layers.7.mlp.experts.linear_fc2.weight3 + module.decoder.layers.7.mlp.experts.linear_fc2.weight2 + module.decoder.layers.7.mlp.experts.linear_fc2.weight0 +Params for bucket 27 (44040192 elements, 44040192 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight8 + module.decoder.layers.7.mlp.experts.linear_fc2.weight6 + module.decoder.layers.7.mlp.experts.linear_fc2.weight10 + module.decoder.layers.7.mlp.experts.linear_fc2.weight5 + module.decoder.layers.7.mlp.experts.linear_fc2.weight9 + module.decoder.layers.7.mlp.experts.linear_fc2.weight7 + module.decoder.layers.7.mlp.experts.linear_fc2.weight11 +Params for bucket 28 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc2.weight12 + module.decoder.layers.7.mlp.experts.linear_fc2.weight14 + module.decoder.layers.7.mlp.experts.linear_fc1.weight1 + module.decoder.layers.7.mlp.experts.linear_fc2.weight15 + module.decoder.layers.7.mlp.experts.linear_fc2.weight13 + module.decoder.layers.7.mlp.experts.linear_fc1.weight0 +Params for bucket 29 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight2 + module.decoder.layers.7.mlp.experts.linear_fc1.weight4 + module.decoder.layers.7.mlp.experts.linear_fc1.weight3 + module.decoder.layers.7.mlp.experts.linear_fc1.weight5 +Params for bucket 30 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight6 + module.decoder.layers.7.mlp.experts.linear_fc1.weight7 + module.decoder.layers.7.mlp.experts.linear_fc1.weight8 + module.decoder.layers.7.mlp.experts.linear_fc1.weight9 +Params for bucket 31 (50331648 elements, 50331648 padded size): + module.decoder.layers.7.mlp.experts.linear_fc1.weight10 + module.decoder.layers.7.mlp.experts.linear_fc1.weight11 + module.decoder.layers.7.mlp.experts.linear_fc1.weight12 + module.decoder.layers.7.mlp.experts.linear_fc1.weight13 +Params for bucket 32 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight0 + module.decoder.layers.7.mlp.experts.linear_fc1.weight15 + module.decoder.layers.7.mlp.experts.linear_fc1.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight2 + module.decoder.layers.6.mlp.experts.linear_fc2.weight1 +Params for bucket 33 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight3 + module.decoder.layers.6.mlp.experts.linear_fc2.weight7 + module.decoder.layers.6.mlp.experts.linear_fc2.weight5 + module.decoder.layers.6.mlp.experts.linear_fc2.weight9 + module.decoder.layers.6.mlp.experts.linear_fc2.weight6 + module.decoder.layers.6.mlp.experts.linear_fc2.weight4 + module.decoder.layers.6.mlp.experts.linear_fc2.weight8 +Params for bucket 34 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc2.weight11 + module.decoder.layers.6.mlp.experts.linear_fc2.weight13 + module.decoder.layers.6.mlp.experts.linear_fc2.weight15 + module.decoder.layers.6.mlp.experts.linear_fc1.weight0 + module.decoder.layers.6.mlp.experts.linear_fc2.weight10 + module.decoder.layers.6.mlp.experts.linear_fc2.weight14 + module.decoder.layers.6.mlp.experts.linear_fc2.weight12 +Params for bucket 35 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight1 + module.decoder.layers.6.mlp.experts.linear_fc1.weight3 + module.decoder.layers.6.mlp.experts.linear_fc1.weight4 + module.decoder.layers.6.mlp.experts.linear_fc1.weight2 +Params for bucket 36 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight5 + module.decoder.layers.6.mlp.experts.linear_fc1.weight6 + module.decoder.layers.6.mlp.experts.linear_fc1.weight8 + module.decoder.layers.6.mlp.experts.linear_fc1.weight7 +Params for bucket 37 (50331648 elements, 50331648 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight9 + module.decoder.layers.6.mlp.experts.linear_fc1.weight12 + module.decoder.layers.6.mlp.experts.linear_fc1.weight10 + module.decoder.layers.6.mlp.experts.linear_fc1.weight11 +Params for bucket 38 (44040192 elements, 44040192 padded size): + module.decoder.layers.6.mlp.experts.linear_fc1.weight13 + module.decoder.layers.6.mlp.experts.linear_fc1.weight15 + module.decoder.layers.5.mlp.experts.linear_fc2.weight0 + module.decoder.layers.6.mlp.experts.linear_fc1.weight14 +Params for bucket 39 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight1 + module.decoder.layers.5.mlp.experts.linear_fc2.weight4 + module.decoder.layers.5.mlp.experts.linear_fc2.weight6 + module.decoder.layers.5.mlp.experts.linear_fc2.weight2 + module.decoder.layers.5.mlp.experts.linear_fc2.weight3 + module.decoder.layers.5.mlp.experts.linear_fc2.weight5 + module.decoder.layers.5.mlp.experts.linear_fc2.weight7 +Params for bucket 40 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight8 + module.decoder.layers.5.mlp.experts.linear_fc2.weight10 + module.decoder.layers.5.mlp.experts.linear_fc2.weight12 + module.decoder.layers.5.mlp.experts.linear_fc2.weight14 + module.decoder.layers.5.mlp.experts.linear_fc2.weight9 + module.decoder.layers.5.mlp.experts.linear_fc2.weight11 + module.decoder.layers.5.mlp.experts.linear_fc2.weight13 +Params for bucket 41 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc2.weight15 + module.decoder.layers.5.mlp.experts.linear_fc1.weight0 + module.decoder.layers.5.mlp.experts.linear_fc1.weight1 + module.decoder.layers.5.mlp.experts.linear_fc1.weight2 +Params for bucket 42 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight3 + module.decoder.layers.5.mlp.experts.linear_fc1.weight4 + module.decoder.layers.5.mlp.experts.linear_fc1.weight6 + module.decoder.layers.5.mlp.experts.linear_fc1.weight5 +Params for bucket 43 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight7 + module.decoder.layers.5.mlp.experts.linear_fc1.weight10 + module.decoder.layers.5.mlp.experts.linear_fc1.weight8 + module.decoder.layers.5.mlp.experts.linear_fc1.weight9 +Params for bucket 44 (50331648 elements, 50331648 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight11 + module.decoder.layers.5.mlp.experts.linear_fc1.weight13 + module.decoder.layers.5.mlp.experts.linear_fc1.weight14 + module.decoder.layers.5.mlp.experts.linear_fc1.weight12 +Params for bucket 45 (44040192 elements, 44040192 padded size): + module.decoder.layers.5.mlp.experts.linear_fc1.weight15 + module.decoder.layers.4.mlp.experts.linear_fc2.weight2 + module.decoder.layers.4.mlp.experts.linear_fc2.weight3 + module.decoder.layers.4.mlp.experts.linear_fc2.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc2.weight4 +Params for bucket 46 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight5 + module.decoder.layers.4.mlp.experts.linear_fc2.weight7 + module.decoder.layers.4.mlp.experts.linear_fc2.weight9 + module.decoder.layers.4.mlp.experts.linear_fc2.weight11 + module.decoder.layers.4.mlp.experts.linear_fc2.weight6 + module.decoder.layers.4.mlp.experts.linear_fc2.weight8 + module.decoder.layers.4.mlp.experts.linear_fc2.weight10 +Params for bucket 47 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc2.weight15 + module.decoder.layers.4.mlp.experts.linear_fc1.weight0 + module.decoder.layers.4.mlp.experts.linear_fc2.weight13 + module.decoder.layers.4.mlp.experts.linear_fc2.weight12 + module.decoder.layers.4.mlp.experts.linear_fc2.weight14 + module.decoder.layers.4.mlp.experts.linear_fc1.weight1 +Params for bucket 48 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight2 + module.decoder.layers.4.mlp.experts.linear_fc1.weight5 + module.decoder.layers.4.mlp.experts.linear_fc1.weight3 + module.decoder.layers.4.mlp.experts.linear_fc1.weight4 +Params for bucket 49 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight6 + module.decoder.layers.4.mlp.experts.linear_fc1.weight8 + module.decoder.layers.4.mlp.experts.linear_fc1.weight9 + module.decoder.layers.4.mlp.experts.linear_fc1.weight7 +Params for bucket 50 (50331648 elements, 50331648 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight10 + module.decoder.layers.4.mlp.experts.linear_fc1.weight12 + module.decoder.layers.4.mlp.experts.linear_fc1.weight13 + module.decoder.layers.4.mlp.experts.linear_fc1.weight11 +Params for bucket 51 (44040192 elements, 44040192 padded size): + module.decoder.layers.4.mlp.experts.linear_fc1.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight2 + module.decoder.layers.3.mlp.experts.linear_fc2.weight1 + module.decoder.layers.4.mlp.experts.linear_fc1.weight15 + module.decoder.layers.3.mlp.experts.linear_fc2.weight0 +Params for bucket 52 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight6 + module.decoder.layers.3.mlp.experts.linear_fc2.weight4 + module.decoder.layers.3.mlp.experts.linear_fc2.weight8 + module.decoder.layers.3.mlp.experts.linear_fc2.weight3 + module.decoder.layers.3.mlp.experts.linear_fc2.weight7 + module.decoder.layers.3.mlp.experts.linear_fc2.weight5 + module.decoder.layers.3.mlp.experts.linear_fc2.weight9 +Params for bucket 53 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc2.weight10 + module.decoder.layers.3.mlp.experts.linear_fc2.weight14 + module.decoder.layers.3.mlp.experts.linear_fc2.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight0 + module.decoder.layers.3.mlp.experts.linear_fc2.weight13 + module.decoder.layers.3.mlp.experts.linear_fc2.weight11 + module.decoder.layers.3.mlp.experts.linear_fc2.weight15 +Params for bucket 54 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight1 + module.decoder.layers.3.mlp.experts.linear_fc1.weight3 + module.decoder.layers.3.mlp.experts.linear_fc1.weight2 + module.decoder.layers.3.mlp.experts.linear_fc1.weight4 +Params for bucket 55 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight5 + module.decoder.layers.3.mlp.experts.linear_fc1.weight8 + module.decoder.layers.3.mlp.experts.linear_fc1.weight6 + module.decoder.layers.3.mlp.experts.linear_fc1.weight7 +Params for bucket 56 (50331648 elements, 50331648 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight9 + module.decoder.layers.3.mlp.experts.linear_fc1.weight12 + module.decoder.layers.3.mlp.experts.linear_fc1.weight10 + module.decoder.layers.3.mlp.experts.linear_fc1.weight11 +Params for bucket 57 (44040192 elements, 44040192 padded size): + module.decoder.layers.3.mlp.experts.linear_fc1.weight13 + module.decoder.layers.3.mlp.experts.linear_fc1.weight15 + module.decoder.layers.2.mlp.experts.linear_fc2.weight0 + module.decoder.layers.3.mlp.experts.linear_fc1.weight14 +Params for bucket 58 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight1 + module.decoder.layers.2.mlp.experts.linear_fc2.weight4 + module.decoder.layers.2.mlp.experts.linear_fc2.weight6 + module.decoder.layers.2.mlp.experts.linear_fc2.weight3 + module.decoder.layers.2.mlp.experts.linear_fc2.weight5 + module.decoder.layers.2.mlp.experts.linear_fc2.weight2 + module.decoder.layers.2.mlp.experts.linear_fc2.weight7 +Params for bucket 59 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight8 + module.decoder.layers.2.mlp.experts.linear_fc2.weight10 + module.decoder.layers.2.mlp.experts.linear_fc2.weight12 + module.decoder.layers.2.mlp.experts.linear_fc2.weight14 + module.decoder.layers.2.mlp.experts.linear_fc2.weight11 + module.decoder.layers.2.mlp.experts.linear_fc2.weight9 + module.decoder.layers.2.mlp.experts.linear_fc2.weight13 +Params for bucket 60 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc2.weight15 + module.decoder.layers.2.mlp.experts.linear_fc1.weight0 + module.decoder.layers.2.mlp.experts.linear_fc1.weight1 + module.decoder.layers.2.mlp.experts.linear_fc1.weight2 +Params for bucket 61 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight3 + module.decoder.layers.2.mlp.experts.linear_fc1.weight4 + module.decoder.layers.2.mlp.experts.linear_fc1.weight6 + module.decoder.layers.2.mlp.experts.linear_fc1.weight5 +Params for bucket 62 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight7 + module.decoder.layers.2.mlp.experts.linear_fc1.weight10 + module.decoder.layers.2.mlp.experts.linear_fc1.weight8 + module.decoder.layers.2.mlp.experts.linear_fc1.weight9 +Params for bucket 63 (50331648 elements, 50331648 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight11 + module.decoder.layers.2.mlp.experts.linear_fc1.weight13 + module.decoder.layers.2.mlp.experts.linear_fc1.weight14 + module.decoder.layers.2.mlp.experts.linear_fc1.weight12 +Params for bucket 64 (44040192 elements, 44040192 padded size): + module.decoder.layers.2.mlp.experts.linear_fc1.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight2 + module.decoder.layers.1.mlp.experts.linear_fc2.weight3 + module.decoder.layers.1.mlp.experts.linear_fc2.weight1 + module.decoder.layers.1.mlp.experts.linear_fc2.weight4 +Params for bucket 65 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc2.weight5 + module.decoder.layers.1.mlp.experts.linear_fc2.weight7 + module.decoder.layers.1.mlp.experts.linear_fc2.weight9 + module.decoder.layers.1.mlp.experts.linear_fc2.weight11 + module.decoder.layers.1.mlp.experts.linear_fc2.weight8 + module.decoder.layers.1.mlp.experts.linear_fc2.weight6 + module.decoder.layers.1.mlp.experts.linear_fc2.weight10 +Params for bucket 66 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight0 + module.decoder.layers.1.mlp.experts.linear_fc2.weight15 + module.decoder.layers.1.mlp.experts.linear_fc2.weight13 + module.decoder.layers.1.mlp.experts.linear_fc2.weight12 + module.decoder.layers.1.mlp.experts.linear_fc2.weight14 + module.decoder.layers.1.mlp.experts.linear_fc1.weight1 +Params for bucket 67 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight2 + module.decoder.layers.1.mlp.experts.linear_fc1.weight4 + module.decoder.layers.1.mlp.experts.linear_fc1.weight5 + module.decoder.layers.1.mlp.experts.linear_fc1.weight3 +Params for bucket 68 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight6 + module.decoder.layers.1.mlp.experts.linear_fc1.weight7 + module.decoder.layers.1.mlp.experts.linear_fc1.weight9 + module.decoder.layers.1.mlp.experts.linear_fc1.weight8 +Params for bucket 69 (50331648 elements, 50331648 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight10 + module.decoder.layers.1.mlp.experts.linear_fc1.weight11 + module.decoder.layers.1.mlp.experts.linear_fc1.weight12 + module.decoder.layers.1.mlp.experts.linear_fc1.weight13 +Params for bucket 70 (44040192 elements, 44040192 padded size): + module.decoder.layers.1.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight2 + module.decoder.layers.0.mlp.experts.linear_fc2.weight0 + module.decoder.layers.1.mlp.experts.linear_fc1.weight15 + module.decoder.layers.0.mlp.experts.linear_fc2.weight1 +Params for bucket 71 (44040192 elements, 44040192 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight3 + module.decoder.layers.0.mlp.experts.linear_fc2.weight7 + module.decoder.layers.0.mlp.experts.linear_fc2.weight5 + module.decoder.layers.0.mlp.experts.linear_fc2.weight9 + module.decoder.layers.0.mlp.experts.linear_fc2.weight6 + module.decoder.layers.0.mlp.experts.linear_fc2.weight4 + module.decoder.layers.0.mlp.experts.linear_fc2.weight8 +Params for bucket 72 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc2.weight13 + module.decoder.layers.0.mlp.experts.linear_fc2.weight12 + module.decoder.layers.0.mlp.experts.linear_fc2.weight15 + module.decoder.layers.0.mlp.experts.linear_fc1.weight0 + module.decoder.layers.0.mlp.experts.linear_fc2.weight10 + module.decoder.layers.0.mlp.experts.linear_fc2.weight14 + module.decoder.layers.0.mlp.experts.linear_fc2.weight11 +Params for bucket 73 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight1 + module.decoder.layers.0.mlp.experts.linear_fc1.weight4 + module.decoder.layers.0.mlp.experts.linear_fc1.weight2 + module.decoder.layers.0.mlp.experts.linear_fc1.weight3 +Params for bucket 74 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight5 + module.decoder.layers.0.mlp.experts.linear_fc1.weight7 + module.decoder.layers.0.mlp.experts.linear_fc1.weight6 + module.decoder.layers.0.mlp.experts.linear_fc1.weight8 +Params for bucket 75 (50331648 elements, 50331648 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight9 + module.decoder.layers.0.mlp.experts.linear_fc1.weight10 + module.decoder.layers.0.mlp.experts.linear_fc1.weight11 + module.decoder.layers.0.mlp.experts.linear_fc1.weight12 +Params for bucket 76 (37748736 elements, 37748736 padded size): + module.decoder.layers.0.mlp.experts.linear_fc1.weight13 + module.decoder.layers.0.mlp.experts.linear_fc1.weight14 + module.decoder.layers.0.mlp.experts.linear_fc1.weight15 +INFO:megatron.core.optimizer:Setting up optimizer with config OptimizerConfig(optimizer='adam', lr=0.0001, min_lr=1e-05, decoupled_lr=None, decoupled_min_lr=None, weight_decay=0.1, fp8_recipe='mxfp8', fp16=False, bf16=True, reuse_grad_buf_for_mxfp8_param_ag=False, params_dtype=torch.bfloat16, use_precision_aware_optimizer=False, store_param_remainders=True, main_grads_dtype=torch.float32, main_params_dtype=torch.float32, exp_avg_dtype=torch.float32, exp_avg_sq_dtype=torch.float32, loss_scale=None, initial_loss_scale=4294967296, min_loss_scale=1.0, loss_scale_window=1000, hysteresis=2, adam_beta1=0.9, adam_beta2=0.999, adam_eps=1e-08, sgd_momentum=0.9, use_distributed_optimizer=True, overlap_param_gather_with_optimizer_step=False, optimizer_cpu_offload=False, optimizer_offload_fraction=1.0, use_torch_optimizer_for_cpu_offload=False, overlap_cpu_optimizer_d2h_h2d=False, pin_cpu_grads=True, pin_cpu_params=True, clip_grad=1.0, log_num_zeros_in_grad=False, barrier_with_L1_time=True, timers=, config_logger_dir='') +INFO:megatron.core.optimizer_param_scheduler:> learning rate decay style: cosine +[after model, optimizer, and learning rate scheduler are built] datetime: 2026-08-29 23:36:00 +> building train, validation, and test datasets ... + > datasets target sizes (minimum size): + train: 96 + validation: 0 + test: 0 +INFO:megatron.core.datasets.blended_megatron_dataset_config:Let split_matrix = [(0, 1.0), None, None] +> building train, validation, and test datasets for GPT ... +INFO:megatron.core.datasets.blended_megatron_dataset_builder:Building GPTDataset splits with sizes=(96, 0, 0) and config=GPTDatasetConfig(random_seed=1234, sequence_length=8192, blend=(['/workspace/data/wikitext/wikitext103_gpt2_text_document'], None), blend_per_split=None, split='100,0,0', split_matrix=[(0, 1.0), None, None], num_dataset_builder_threads=1, path_to_cache=None, mmap_bin_files=True, mock=False, tokenizer=, mid_level_dataset_surplus=0.005, reset_position_ids=False, reset_attention_mask=False, eod_mask_loss=False, create_attention_mask=True, drop_last_partial_validation_sequence=True, add_extra_token_to_sequence=True, object_storage_cache_path=None) +INFO:megatron.core.datasets.indexed_dataset:Load the _IndexReader from /workspace/data/wikitext/wikitext103_gpt2_text_document.idx +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence lengths +INFO:megatron.core.datasets.indexed_dataset: Extract the sequence pointers +INFO:megatron.core.datasets.indexed_dataset: Extract the document indices +INFO:megatron.core.datasets.indexed_dataset:> total number of sequences: 1165029 +INFO:megatron.core.datasets.indexed_dataset:> total number of documents: 1801350 +INFO:megatron.core.datasets.gpt_dataset:Load the GPTDataset train indices +INFO:megatron.core.datasets.gpt_dataset: Load the document index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-document_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the sample index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-sample_index.npy +INFO:megatron.core.datasets.gpt_dataset: Load the shuffle index from eadc3756447e39d609e8aa728a91f451-GPTDataset-train-shuffle_index.npy +INFO:megatron.core.datasets.gpt_dataset:> total number of samples: 14536 +> finished creating GPT datasets ... +[after dataloaders are built] datetime: 2026-08-29 23:36:00 +done with setup ... +(min, max) time across ranks (ms): + model-and-optimizer-setup ......................: (3286.88, 3307.50) + train/valid/test-data-iterators-setup ..........: (64.74, 72.68) +training ... +Overwriting rerun_state_machine.current_iteration from -1 to 0... +[before the start of training step] datetime: 2026-08-29 23:36:00 +INFO:moonep.api:MoonEP comm buffer sizes: + config : R=8, S=8192, H=4096, E=128, K=8, NvS=69600, NvS_padded=69632 + sm_count : num_sms=32, num_sms_dedup=148 + hidden_buffer_size : 544.00 MiB + meta_buffer_size : 2.00 MiB + local_temp_buffer_size : 2.52 MiB + total_size : 548.52 MiB +[MoonEP selfcheck 1/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [4096, 4096, 4224, 4096, 4224, 4096, 4224, 4224, 4224, 4096, 4096, 4224, 4224, 4096, 4096, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[79, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [44, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [125, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [116, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [114, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [36, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 2/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [4224, 4224, 4096, 4096, 4096, 4096, 4224, 4224, 4224, 4096, 4096, 4096, 4096, 4224, 4096, 4224, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [13, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [73, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [2, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [0, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [28, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [12, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [60, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 3/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [4224, 4096, 4096, 4096, 4224, 4096, 4224, 4096, 4224, 4096, 3968, 4224, 4224, 4224, 4224, 4224, 128, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [46, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [41, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [8, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [66, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [51, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [74, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +[MoonEP selfcheck 4/4] dropped groups empty OK | covered rows [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143] sizes [4224, 4224, 4096, 4224, 4096, 4224, 4096, 4096, 4096, 4224, 4224, 4096, 4224, 4096, 4096, 4224, 256, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] | duplicated experts [[95, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [65, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [77, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [25, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [87, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [70, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1], [81, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1]] +WARNING:megatron.core.rerun_state_machine:Result validation enabled + [2026-08-29 23:36:22] iteration 1/ 12 | consumed samples: 8 | elapsed time per iteration (ms): 21337.5 | throughput per GPU (TFLOP/s/GPU): 8.5 | learning rate: 2.000000E-05 | global batch size: 8 | lm loss: 1.115718E+01 | loss scale: 1.0 | grad norm: 44.806 | number of skipped iterations: 0 | number of nan iterations: 0 | +Number of parameters in transformer block in billions: 29.85 +Number of parameters in embedding layers in billions: 0.41 +Total number of parameters in billions: 30.26 +Number of parameters in most loaded shard in billions: 30.2590 +Theoretical memory footprints: weight and optimizer=216428.96 MB +[Rank 0] (after 1 iterations) memory (MB) | allocated: 67174.95703125 | max allocated: 67174.95751953125 | reserved: 68426.0 | max reserved: 68426.0 + [2026-08-29 23:36:22] iteration 2/ 12 | consumed samples: 16 | elapsed time per iteration (ms): 405.1 | throughput per GPU (TFLOP/s/GPU): 446.3 | learning rate: 4.000000E-05 | global batch size: 8 | lm loss: 1.117266E+01 | loss scale: 1.0 | grad norm: 44.201 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87316 payload_rows_remote=76408 dup_rows=43756 dup_rows_remote=38328 dedup_ratio=0.333832 router_tokens_per_expert=[501, 500, 526, 483, 534, 511, 500, 528, 515, 514, 515, 552, 513, 517, 480, 521, 494, 514, 509, 513, 501, 474, 519, 520, 490, 526, 556, 474, 523, 511, 549, 495, 518, 525, 508, 524, 524, 493, 508, 497, 509, 528, 511, 493, 505, 498, 468, 492, 506, 523, 526, 512, 516, 549, 489, 497, 514, 512, 488, 509, 515, 489, 478, 531, 485, 507, 487, 501, 514, 512, 517, 486, 484, 525, 532, 490, 500, 509, 501, 510, 548, 519, 491, 539, 525, 525, 554, 493, 522, 503, 519, 532, 528, 506, 554, 540, 536, 547, 534, 523, 535, 527, 502, 467, 507, 513, 486, 483, 508, 500, 532, 542, 508, 490, 548, 512, 513, 508, 514, 514, 503, 513, 482, 540, 544, 535, 494, 485] payload_rows_per_dst=[10940, 10908, 10830, 10862, 10776, 11038, 10970, 10992] dup_rows_per_dst=[5480, 5428, 5372, 5446, 5344, 5758, 5514, 5414] combine_rows_per_src=[10940, 10908, 10830, 10862, 10776, 11038, 10970, 10992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87386 payload_rows_remote=76466 dup_rows=43686 dup_rows_remote=38264 dedup_ratio=0.333298 router_tokens_per_expert=[526, 549, 535, 507, 521, 472, 554, 472, 516, 501, 515, 518, 539, 516, 527, 512, 495, 521, 484, 529, 501, 487, 525, 529, 515, 515, 523, 520, 506, 502, 486, 508, 486, 545, 510, 480, 502, 497, 534, 486, 498, 525, 493, 532, 479, 477, 523, 509, 531, 472, 537, 506, 497, 493, 521, 511, 531, 471, 561, 554, 481, 485, 478, 542, 499, 509, 501, 537, 502, 488, 544, 493, 533, 539, 476, 495, 522, 522, 509, 493, 539, 538, 506, 515, 530, 492, 528, 475, 493, 491, 517, 516, 490, 516, 515, 517, 528, 551, 498, 501, 498, 498, 479, 529, 503, 504, 517, 519, 521, 533, 513, 506, 517, 506, 510, 524, 548, 482, 551, 541, 536, 520, 512, 499, 496, 498, 538, 547] payload_rows_per_dst=[11012, 10824, 10830, 10920, 10910, 10854, 11002, 11034] dup_rows_per_dst=[5548, 5468, 5322, 5422, 5414, 5502, 5394, 5616] combine_rows_per_src=[11012, 10824, 10830, 10920, 10910, 10854, 11002, 11034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87550 payload_rows_remote=76524 dup_rows=43522 dup_rows_remote=37894 dedup_ratio=0.332047 router_tokens_per_expert=[529, 547, 520, 516, 495, 505, 543, 495, 553, 524, 513, 485, 471, 520, 515, 507, 531, 516, 541, 502, 529, 522, 530, 497, 500, 505, 523, 558, 494, 533, 527, 460, 493, 528, 497, 473, 546, 523, 557, 535, 509, 501, 504, 511, 556, 567, 487, 540, 501, 495, 522, 483, 502, 509, 530, 487, 511, 472, 539, 465, 516, 547, 491, 486, 507, 522, 530, 530, 558, 504, 504, 524, 498, 516, 540, 501, 511, 499, 502, 478, 506, 512, 510, 484, 475, 512, 532, 473, 514, 510, 535, 565, 488, 488, 541, 534, 480, 531, 509, 527, 465, 521, 484, 484, 502, 508, 509, 538, 519, 511, 490, 521, 514, 495, 501, 514, 476, 505, 541, 510, 510, 508, 551, 494, 495, 489, 537, 505] payload_rows_per_dst=[10954, 11002, 11026, 10772, 10902, 11074, 10900, 10920] dup_rows_per_dst=[5522, 5534, 5628, 5340, 5546, 5284, 5298, 5370] combine_rows_per_src=[10954, 11002, 11026, 10772, 10902, 11074, 10900, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87500 payload_rows_remote=76590 dup_rows=43572 dup_rows_remote=38088 dedup_ratio=0.332428 router_tokens_per_expert=[519, 524, 499, 544, 511, 504, 509, 498, 519, 495, 515, 518, 522, 489, 502, 499, 522, 493, 538, 508, 544, 517, 528, 473, 494, 523, 487, 515, 512, 531, 473, 506, 512, 483, 511, 518, 517, 539, 500, 508, 542, 519, 527, 548, 495, 495, 496, 522, 519, 507, 523, 483, 488, 500, 484, 506, 535, 541, 513, 501, 465, 523, 527, 497, 513, 516, 494, 497, 530, 496, 484, 501, 504, 512, 515, 532, 484, 510, 475, 509, 537, 507, 516, 495, 478, 503, 474, 514, 540, 577, 458, 537, 542, 480, 533, 506, 543, 547, 544, 489, 487, 477, 525, 543, 520, 492, 522, 527, 549, 569, 482, 530, 552, 498, 534, 512, 545, 506, 492, 493, 496, 517, 509, 518, 508, 527, 536, 503] payload_rows_per_dst=[10852, 10910, 10940, 10930, 10920, 10910, 11048, 10990] dup_rows_per_dst=[5482, 5418, 5524, 5294, 5224, 5484, 5644, 5502] combine_rows_per_src=[10852, 10910, 10940, 10930, 10920, 10910, 11048, 10990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87722 payload_rows_remote=76722 dup_rows=43350 dup_rows_remote=37650 dedup_ratio=0.330734 router_tokens_per_expert=[483, 477, 491, 507, 521, 473, 536, 467, 493, 497, 467, 475, 513, 537, 469, 482, 520, 509, 524, 515, 474, 515, 532, 506, 462, 533, 524, 572, 497, 528, 544, 473, 505, 528, 509, 481, 532, 474, 521, 507, 511, 525, 546, 511, 514, 546, 458, 495, 501, 471, 491, 487, 456, 497, 516, 514, 569, 501, 510, 515, 512, 521, 524, 516, 542, 544, 557, 526, 530, 498, 525, 494, 506, 532, 509, 470, 499, 519, 524, 505, 527, 498, 517, 566, 519, 505, 524, 559, 485, 521, 534, 531, 493, 484, 489, 517, 553, 493, 502, 490, 509, 538, 521, 555, 522, 522, 520, 548, 503, 515, 466, 502, 493, 513, 492, 528, 521, 509, 551, 586, 543, 517, 554, 523, 500, 524, 486, 508] payload_rows_per_dst=[10688, 10936, 10984, 10894, 11092, 11006, 11122, 11000] dup_rows_per_dst=[5088, 5516, 5284, 5308, 5468, 5450, 5536, 5700] combine_rows_per_src=[10688, 10936, 10984, 10894, 11092, 11006, 11122, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87662 payload_rows_remote=76524 dup_rows=43410 dup_rows_remote=37630 dedup_ratio=0.331192 router_tokens_per_expert=[504, 564, 487, 453, 516, 524, 529, 499, 497, 498, 508, 518, 506, 508, 485, 501, 519, 466, 498, 529, 466, 478, 477, 542, 545, 502, 519, 516, 518, 528, 488, 528, 464, 500, 520, 549, 526, 533, 522, 548, 500, 505, 501, 511, 539, 481, 516, 515, 537, 496, 491, 502, 504, 547, 519, 511, 488, 508, 519, 490, 529, 485, 531, 527, 497, 498, 488, 497, 524, 496, 510, 504, 553, 506, 475, 521, 512, 485, 512, 504, 557, 491, 473, 514, 487, 494, 504, 566, 524, 530, 522, 512, 500, 505, 534, 509, 498, 524, 505, 524, 521, 554, 523, 523, 503, 531, 543, 532, 510, 472, 515, 509, 497, 557, 532, 516, 530, 517, 486, 525, 533, 525, 527, 497, 526, 521, 494, 532] payload_rows_per_dst=[11138, 10876, 11010, 11346, 11034, 10722, 10942, 10594] dup_rows_per_dst=[5780, 5362, 5462, 5930, 5562, 4998, 5200, 5116] combine_rows_per_src=[11138, 10876, 11010, 11346, 11034, 10722, 10942, 10594] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=458944 combine_payload_bytes=625934336 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=626393280 ingress_bytes=701431808 link_bytes=701431808 token_bytes=1252327616 weight_movement_bytes=75497472 total_nvlink_bytes=1327825088 weight_to_token_ratio=0.060286 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895142665e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626409472 dispatch_meta_bytes=458920 combine_payload_bytes=626409472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626868392 ingress_bytes=701906944 link_bytes=701906944 token_bytes=1253277864 weight_movement_bytes=75497472 total_nvlink_bytes=1328775336 weight_to_token_ratio=0.060240 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901168059e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=457672 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627342280 ingress_bytes=777879552 link_bytes=777879552 token_bytes=1254226888 weight_movement_bytes=150994944 total_nvlink_bytes=1405221832 weight_to_token_ratio=0.120389 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864607668e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87246 payload_rows_remote=76278 dup_rows=43826 dup_rows_remote=38324 dedup_ratio=0.334366 router_tokens_per_expert=[479, 489, 535, 482, 514, 497, 522, 507, 512, 520, 532, 485, 500, 515, 486, 501, 526, 496, 492, 505, 495, 507, 565, 525, 491, 528, 476, 526, 513, 559, 526, 509, 512, 515, 511, 540, 531, 538, 506, 489, 493, 503, 519, 538, 516, 537, 510, 528, 512, 518, 514, 492, 507, 515, 479, 534, 504, 538, 453, 478, 514, 528, 547, 499, 512, 581, 505, 532, 475, 487, 490, 555, 501, 520, 528, 455, 500, 568, 457, 527, 495, 497, 479, 548, 517, 507, 545, 499, 540, 529, 496, 520, 522, 492, 514, 476, 510, 473, 507, 552, 475, 487, 525, 557, 514, 513, 530, 532, 536, 518, 501, 505, 515, 543, 487, 512, 517, 518, 533, 485, 516, 506, 483, 477, 559, 510, 514, 524] payload_rows_per_dst=[10854, 10926, 10854, 10936, 10936, 10852, 10968, 10920] dup_rows_per_dst=[5298, 5552, 5718, 5328, 5450, 5500, 5502, 5478] combine_rows_per_src=[10854, 10926, 10854, 10936, 10936, 10852, 10968, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627425280 dispatch_meta_bytes=458712 combine_payload_bytes=627425280 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627883992 ingress_bytes=853917696 link_bytes=853917696 token_bytes=1255309272 weight_movement_bytes=226492416 total_nvlink_bytes=1481801688 weight_to_token_ratio=0.180428 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082887837e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628506624 dispatch_meta_bytes=457488 combine_payload_bytes=628506624 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628964112 ingress_bytes=854999040 link_bytes=854999040 token_bytes=1257470736 weight_movement_bytes=226492416 total_nvlink_bytes=1483963152 weight_to_token_ratio=0.180117 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084259133e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=456616 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627341224 ingress_bytes=702382080 link_bytes=702382080 token_bytes=1254225832 weight_movement_bytes=75497472 total_nvlink_bytes=1329723304 weight_to_token_ratio=0.060194 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907193452e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87276 payload_rows_remote=76344 dup_rows=43796 dup_rows_remote=38234 dedup_ratio=0.334137 router_tokens_per_expert=[518, 489, 504, 511, 495, 507, 533, 542, 522, 533, 505, 523, 493, 516, 524, 503, 506, 531, 493, 488, 511, 535, 526, 510, 465, 505, 528, 554, 517, 506, 490, 541, 468, 503, 519, 497, 552, 525, 491, 524, 515, 526, 513, 513, 511, 499, 483, 505, 495, 524, 490, 530, 498, 515, 502, 490, 538, 512, 498, 527, 541, 507, 507, 498, 473, 466, 528, 539, 544, 510, 546, 487, 531, 549, 534, 502, 510, 498, 522, 508, 520, 499, 526, 519, 520, 529, 506, 542, 500, 473, 508, 558, 553, 460, 543, 564, 484, 496, 501, 525, 494, 509, 529, 489, 475, 506, 495, 525, 519, 506, 488, 475, 511, 516, 513, 491, 543, 540, 497, 509, 474, 502, 502, 525, 507, 517, 559, 507] payload_rows_per_dst=[10874, 11030, 10826, 10864, 10932, 10964, 10868, 10918] dup_rows_per_dst=[5562, 5382, 5462, 5480, 5562, 5676, 5164, 5508] combine_rows_per_src=[10874, 11030, 10826, 10864, 10932, 10964, 10868, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87704 payload_rows_remote=76824 dup_rows=43368 dup_rows_remote=38058 dedup_ratio=0.330872 router_tokens_per_expert=[472, 539, 549, 535, 519, 506, 500, 519, 506, 527, 478, 520, 504, 521, 474, 492, 497, 501, 481, 538, 518, 507, 546, 485, 515, 536, 486, 518, 506, 451, 506, 504, 520, 480, 547, 563, 486, 531, 502, 516, 474, 505, 533, 502, 519, 480, 520, 540, 485, 475, 552, 504, 487, 518, 499, 523, 518, 490, 505, 484, 500, 526, 519, 543, 547, 522, 488, 496, 532, 519, 503, 526, 546, 488, 520, 548, 506, 506, 457, 489, 512, 507, 503, 529, 494, 523, 499, 516, 520, 553, 549, 492, 543, 514, 507, 521, 505, 482, 535, 526, 540, 495, 500, 527, 523, 510, 507, 513, 523, 524, 508, 541, 530, 514, 500, 506, 532, 505, 497, 505, 525, 516, 507, 524, 511, 483, 501, 544] payload_rows_per_dst=[10798, 10880, 10968, 10882, 11052, 11058, 11064, 11002] dup_rows_per_dst=[5524, 5310, 5468, 5374, 5334, 5506, 5454, 5398] combine_rows_per_src=[10798, 10880, 10968, 10882, 11052, 11058, 11064, 11002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87376 payload_rows_remote=76396 dup_rows=43696 dup_rows_remote=38328 dedup_ratio=0.333374 router_tokens_per_expert=[489, 503, 516, 494, 505, 523, 530, 528, 565, 472, 492, 508, 566, 538, 529, 521, 491, 527, 506, 528, 500, 532, 537, 534, 492, 490, 522, 483, 535, 515, 492, 531, 474, 496, 516, 500, 523, 502, 554, 498, 544, 526, 510, 533, 535, 469, 514, 480, 491, 504, 462, 504, 495, 475, 508, 475, 512, 498, 506, 506, 516, 502, 499, 508, 502, 490, 524, 520, 516, 481, 472, 549, 494, 481, 501, 537, 519, 533, 552, 516, 519, 522, 499, 533, 493, 519, 537, 525, 549, 543, 516, 504, 515, 516, 525, 510, 520, 493, 512, 501, 485, 499, 523, 511, 495, 537, 546, 478, 508, 526, 472, 555, 471, 502, 564, 524, 522, 507, 514, 495, 519, 529, 518, 513, 517, 519, 503, 517] payload_rows_per_dst=[11054, 10896, 10980, 10820, 10844, 10940, 10832, 11010] dup_rows_per_dst=[5504, 5534, 5368, 5102, 5530, 5710, 5490, 5458] combine_rows_per_src=[11054, 10896, 10980, 10820, 10844, 10940, 10832, 11010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87664 payload_rows_remote=76720 dup_rows=43408 dup_rows_remote=37982 dedup_ratio=0.331177 router_tokens_per_expert=[490, 478, 523, 502, 527, 491, 518, 512, 521, 455, 510, 477, 526, 498, 479, 505, 527, 495, 504, 517, 473, 500, 514, 516, 528, 556, 486, 521, 504, 503, 532, 498, 512, 490, 484, 526, 544, 550, 515, 539, 494, 537, 500, 539, 509, 500, 518, 510, 534, 503, 476, 505, 525, 507, 503, 544, 493, 541, 515, 507, 526, 487, 502, 517, 516, 501, 517, 486, 527, 521, 524, 555, 542, 510, 503, 551, 516, 519, 518, 551, 503, 492, 528, 526, 523, 522, 505, 544, 528, 501, 530, 544, 505, 500, 481, 528, 491, 481, 521, 501, 485, 510, 495, 482, 518, 491, 518, 512, 504, 506, 501, 531, 526, 495, 522, 513, 491, 526, 520, 499, 518, 503, 518, 528, 481, 544, 499, 551] payload_rows_per_dst=[10844, 10984, 10974, 10944, 11006, 11060, 10874, 10978] dup_rows_per_dst=[5180, 5364, 5560, 5426, 5708, 5460, 5220, 5490] combine_rows_per_src=[10844, 10984, 10974, 10944, 11006, 11060, 10874, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458896 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626294928 ingress_bytes=776830976 link_bytes=776830976 token_bytes=1252130960 weight_movement_bytes=150994944 total_nvlink_bytes=1403125904 weight_to_token_ratio=0.120590 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851310248e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=629342208 dispatch_meta_bytes=459528 combine_payload_bytes=629342208 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629801736 ingress_bytes=780337152 link_bytes=780337152 token_bytes=1259143944 weight_movement_bytes=150994944 total_nvlink_bytes=1410138888 weight_to_token_ratio=0.119919 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.895773495e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628490240 dispatch_meta_bytes=458808 combine_payload_bytes=628490240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628949048 ingress_bytes=703987712 link_bytes=703987712 token_bytes=1257439288 weight_movement_bytes=75497472 total_nvlink_bytes=1332936760 weight_to_token_ratio=0.060041 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927555126e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=624869376 dispatch_meta_bytes=458408 combine_payload_bytes=624869376 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625327784 ingress_bytes=775864320 link_bytes=775864320 token_bytes=1250197160 weight_movement_bytes=150994944 total_nvlink_bytes=1401192104 weight_to_token_ratio=0.120777 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.839051689e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625410048 dispatch_meta_bytes=458312 combine_payload_bytes=625410048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625868360 ingress_bytes=700907520 link_bytes=700907520 token_bytes=1251278408 weight_movement_bytes=75497472 total_nvlink_bytes=1326775880 weight_to_token_ratio=0.060336 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888493956e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87662 payload_rows_remote=76504 dup_rows=43410 dup_rows_remote=37420 dedup_ratio=0.331192 router_tokens_per_expert=[499, 508, 510, 539, 486, 519, 511, 574, 507, 498, 536, 518, 466, 469, 464, 480, 537, 530, 507, 475, 558, 520, 479, 509, 498, 502, 549, 501, 515, 543, 560, 490, 496, 490, 514, 476, 516, 527, 503, 487, 506, 540, 487, 460, 564, 483, 523, 519, 507, 495, 502, 548, 546, 501, 534, 502, 510, 539, 506, 529, 479, 517, 510, 481, 527, 497, 546, 507, 541, 540, 500, 506, 496, 501, 497, 526, 545, 516, 530, 526, 516, 544, 495, 495, 505, 494, 508, 504, 474, 477, 520, 510, 519, 508, 501, 510, 522, 513, 532, 490, 509, 507, 485, 522, 531, 508, 463, 554, 528, 520, 519, 474, 517, 538, 516, 556, 490, 507, 525, 530, 532, 497, 505, 497, 491, 562, 543, 518] payload_rows_per_dst=[11158, 11168, 10594, 11026, 10838, 10870, 11050, 10958] dup_rows_per_dst=[5990, 5420, 4958, 5386, 5512, 5150, 5670, 5324] combine_rows_per_src=[11158, 11168, 10594, 11026, 10838, 10870, 11050, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87408 payload_rows_remote=76384 dup_rows=43664 dup_rows_remote=38226 dedup_ratio=0.333130 router_tokens_per_expert=[469, 516, 544, 539, 500, 524, 514, 465, 501, 497, 495, 527, 531, 520, 534, 485, 522, 517, 518, 486, 509, 485, 503, 486, 503, 545, 512, 522, 500, 513, 542, 490, 558, 531, 542, 518, 537, 504, 529, 524, 518, 517, 511, 514, 530, 518, 507, 516, 504, 526, 540, 492, 532, 505, 516, 504, 541, 524, 487, 571, 546, 517, 502, 513, 505, 499, 494, 496, 521, 544, 482, 487, 488, 485, 467, 501, 512, 530, 490, 528, 492, 504, 496, 529, 520, 471, 550, 541, 489, 514, 524, 533, 524, 520, 483, 524, 483, 510, 513, 518, 514, 474, 493, 497, 527, 504, 498, 515, 535, 502, 511, 498, 504, 472, 494, 522, 537, 519, 567, 525, 478, 546, 461, 530, 500, 525, 509, 504] payload_rows_per_dst=[11004, 10748, 11050, 11088, 10700, 10972, 10822, 11024] dup_rows_per_dst=[5318, 5076, 5698, 6034, 5358, 5380, 5362, 5438] combine_rows_per_src=[11004, 10748, 11050, 11088, 10700, 10972, 10822, 11024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87342 payload_rows_remote=76424 dup_rows=43730 dup_rows_remote=38354 dedup_ratio=0.333633 router_tokens_per_expert=[514, 508, 514, 472, 490, 520, 543, 475, 521, 500, 510, 483, 511, 475, 503, 505, 522, 525, 558, 499, 559, 504, 565, 487, 516, 527, 492, 516, 514, 519, 516, 510, 525, 487, 503, 490, 511, 492, 521, 544, 527, 538, 513, 531, 499, 466, 516, 552, 491, 497, 521, 508, 543, 464, 481, 527, 488, 502, 499, 528, 505, 523, 514, 500, 511, 538, 470, 510, 475, 495, 535, 516, 549, 482, 522, 512, 506, 536, 535, 500, 506, 509, 510, 527, 524, 509, 484, 476, 510, 526, 507, 488, 562, 516, 487, 506, 494, 518, 500, 512, 555, 498, 528, 496, 531, 525, 527, 504, 490, 491, 532, 516, 534, 511, 509, 516, 505, 531, 527, 531, 467, 538, 519, 527, 552, 553, 477, 504] payload_rows_per_dst=[10714, 11052, 10938, 10810, 10918, 10918, 10916, 11076] dup_rows_per_dst=[5374, 5606, 5492, 5372, 5466, 5376, 5518, 5526] combine_rows_per_src=[10714, 11052, 10938, 10810, 10918, 10918, 10916, 11076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87392 payload_rows_remote=76548 dup_rows=43680 dup_rows_remote=38342 dedup_ratio=0.333252 router_tokens_per_expert=[536, 535, 534, 499, 555, 512, 553, 538, 504, 509, 520, 529, 506, 531, 490, 493, 504, 498, 526, 530, 517, 508, 513, 505, 525, 534, 505, 508, 522, 547, 585, 524, 495, 492, 519, 492, 523, 503, 486, 530, 500, 510, 491, 499, 540, 498, 522, 491, 507, 477, 535, 507, 521, 525, 499, 560, 542, 531, 503, 508, 509, 475, 518, 510, 533, 536, 489, 524, 500, 562, 492, 526, 531, 493, 549, 477, 511, 519, 506, 517, 515, 545, 474, 508, 480, 513, 475, 475, 547, 506, 486, 516, 512, 490, 538, 502, 467, 474, 515, 514, 527, 534, 533, 512, 510, 480, 486, 458, 485, 513, 482, 499, 509, 496, 529, 534, 460, 554, 503, 510, 480, 546, 495, 536, 504, 520, 526, 485] payload_rows_per_dst=[10954, 11020, 10844, 10962, 11046, 10934, 10746, 10886] dup_rows_per_dst=[5734, 5682, 5338, 5492, 5484, 5230, 5232, 5488] combine_rows_per_src=[10954, 11020, 10844, 10962, 11046, 10934, 10746, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87360 payload_rows_remote=76498 dup_rows=43712 dup_rows_remote=38208 dedup_ratio=0.333496 router_tokens_per_expert=[511, 507, 506, 535, 521, 553, 509, 552, 549, 477, 482, 515, 493, 496, 498, 512, 494, 509, 475, 489, 517, 506, 540, 497, 518, 543, 522, 480, 534, 507, 527, 525, 536, 514, 543, 489, 519, 510, 529, 518, 507, 503, 491, 512, 501, 522, 500, 487, 512, 490, 502, 497, 526, 523, 502, 491, 499, 499, 458, 526, 533, 517, 508, 509, 494, 487, 513, 508, 511, 515, 482, 494, 525, 524, 489, 476, 501, 521, 494, 523, 489, 538, 511, 539, 493, 473, 546, 511, 479, 510, 518, 495, 508, 512, 549, 521, 536, 484, 504, 547, 502, 528, 502, 531, 482, 506, 526, 511, 511, 551, 543, 524, 528, 557, 514, 522, 522, 556, 507, 499, 483, 511, 495, 535, 512, 537, 508, 541] payload_rows_per_dst=[10984, 10862, 10876, 10756, 10816, 10964, 11020, 11082] dup_rows_per_dst=[5448, 5504, 5486, 5428, 5298, 5420, 5556, 5572] combine_rows_per_src=[10984, 10862, 10876, 10756, 10816, 10964, 11020, 11082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=455696 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627176464 ingress_bytes=702218240 link_bytes=702218240 token_bytes=1253897232 weight_movement_bytes=75497472 total_nvlink_bytes=1329394704 weight_to_token_ratio=0.060210 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905115730e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625737728 dispatch_meta_bytes=458440 combine_payload_bytes=625737728 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626196168 ingress_bytes=776732672 link_bytes=776732672 token_bytes=1251933896 weight_movement_bytes=150994944 total_nvlink_bytes=1402928840 weight_to_token_ratio=0.120609 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850063615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=459112 combine_payload_bytes=626065408 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626524520 ingress_bytes=852557824 link_bytes=852557824 token_bytes=1252589928 weight_movement_bytes=226492416 total_nvlink_bytes=1479082344 weight_to_token_ratio=0.180819 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081163327e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76652 dup_rows=43570 dup_rows_remote=38254 dedup_ratio=0.332413 router_tokens_per_expert=[512, 519, 521, 531, 561, 495, 514, 506, 498, 507, 532, 550, 501, 530, 538, 474, 496, 535, 502, 521, 459, 524, 521, 519, 494, 516, 482, 522, 494, 511, 499, 544, 538, 538, 501, 518, 483, 497, 529, 497, 513, 538, 535, 483, 535, 505, 462, 533, 498, 511, 476, 572, 529, 488, 458, 513, 517, 458, 516, 509, 505, 536, 491, 506, 537, 522, 495, 512, 515, 470, 507, 518, 506, 530, 526, 505, 536, 490, 510, 529, 544, 533, 480, 504, 519, 500, 513, 508, 515, 490, 504, 526, 491, 500, 562, 487, 529, 499, 535, 524, 517, 506, 487, 484, 511, 527, 521, 516, 522, 520, 516, 465, 527, 491, 508, 486, 515, 521, 520, 531, 513, 527, 515, 502, 552, 534, 506, 509] payload_rows_per_dst=[10980, 10922, 10948, 10850, 10952, 10910, 10884, 11056] dup_rows_per_dst=[5598, 5356, 5462, 5316, 5464, 5442, 5474, 5458] combine_rows_per_src=[10980, 10922, 10948, 10850, 10952, 10910, 10884, 11056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627081216 dispatch_meta_bytes=459560 combine_payload_bytes=627081216 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627540776 ingress_bytes=702578688 link_bytes=702578688 token_bytes=1254621992 weight_movement_bytes=75497472 total_nvlink_bytes=1330119464 weight_to_token_ratio=0.060175 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909686718e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626671616 dispatch_meta_bytes=458824 combine_payload_bytes=626671616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627130440 ingress_bytes=702169088 link_bytes=702169088 token_bytes=1253802056 weight_movement_bytes=75497472 total_nvlink_bytes=1329299528 weight_to_token_ratio=0.060215 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904492414e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627933184 dispatch_meta_bytes=459624 combine_payload_bytes=627933184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628392808 ingress_bytes=703430656 link_bytes=703430656 token_bytes=1256325992 weight_movement_bytes=75497472 total_nvlink_bytes=1331823464 weight_to_token_ratio=0.060094 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920490872e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87238 payload_rows_remote=76310 dup_rows=43834 dup_rows_remote=38228 dedup_ratio=0.334427 router_tokens_per_expert=[495, 517, 514, 514, 511, 495, 498, 497, 526, 521, 499, 519, 523, 505, 482, 480, 532, 521, 527, 514, 522, 506, 492, 506, 513, 519, 507, 519, 555, 497, 550, 535, 527, 496, 508, 541, 519, 473, 543, 529, 516, 527, 484, 533, 502, 532, 535, 503, 497, 537, 506, 540, 549, 502, 515, 485, 527, 496, 495, 502, 541, 504, 488, 523, 521, 563, 479, 522, 484, 547, 466, 528, 503, 539, 527, 531, 511, 529, 519, 498, 496, 500, 525, 529, 521, 524, 540, 525, 507, 525, 506, 510, 487, 483, 504, 515, 537, 516, 514, 475, 494, 512, 506, 551, 516, 526, 505, 526, 532, 515, 509, 514, 479, 476, 526, 480, 530, 489, 492, 507, 450, 492, 531, 529, 498, 511, 464, 484] payload_rows_per_dst=[10878, 11012, 10980, 10844, 10928, 10948, 10908, 10740] dup_rows_per_dst=[5314, 5618, 5556, 5570, 5606, 5446, 5588, 5136] combine_rows_per_src=[10878, 11012, 10980, 10844, 10928, 10948, 10908, 10740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76620 dup_rows=43598 dup_rows_remote=38148 dedup_ratio=0.332626 router_tokens_per_expert=[530, 531, 518, 510, 526, 520, 496, 514, 542, 477, 527, 516, 474, 495, 505, 528, 540, 504, 487, 473, 506, 498, 548, 513, 500, 512, 553, 503, 526, 512, 528, 499, 515, 550, 541, 490, 535, 477, 487, 520, 524, 488, 504, 504, 510, 522, 536, 541, 471, 511, 511, 533, 538, 509, 492, 508, 554, 514, 536, 511, 520, 487, 497, 505, 511, 484, 523, 528, 529, 484, 513, 489, 501, 484, 532, 532, 521, 519, 488, 498, 520, 499, 545, 486, 520, 533, 482, 502, 515, 492, 491, 530, 508, 509, 511, 496, 485, 532, 484, 497, 551, 491, 508, 472, 542, 499, 521, 538, 512, 496, 519, 505, 502, 515, 522, 518, 492, 541, 543, 500, 488, 545, 504, 526, 507, 500, 553, 501] payload_rows_per_dst=[10944, 10994, 10954, 10970, 10896, 10872, 10854, 10990] dup_rows_per_dst=[5474, 5410, 5534, 5424, 5376, 5406, 5450, 5524] combine_rows_per_src=[10944, 10994, 10954, 10970, 10896, 10872, 10854, 10990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87252 payload_rows_remote=76406 dup_rows=43820 dup_rows_remote=38400 dedup_ratio=0.334320 router_tokens_per_expert=[540, 501, 535, 498, 504, 491, 479, 460, 452, 558, 524, 501, 472, 511, 557, 539, 513, 527, 518, 503, 501, 503, 548, 512, 529, 475, 508, 543, 487, 487, 486, 542, 499, 478, 496, 534, 549, 502, 512, 503, 542, 527, 529, 546, 491, 528, 485, 532, 512, 517, 493, 505, 517, 490, 513, 520, 483, 537, 509, 508, 489, 513, 530, 462, 514, 513, 505, 519, 472, 524, 522, 528, 521, 516, 524, 486, 490, 503, 541, 527, 520, 487, 530, 512, 513, 511, 494, 515, 492, 522, 526, 517, 481, 507, 506, 522, 535, 519, 551, 537, 531, 493, 531, 504, 509, 520, 532, 514, 556, 516, 533, 507, 524, 495, 507, 504, 514, 480, 487, 522, 538, 466, 514, 526, 548, 505, 505, 498] payload_rows_per_dst=[10536, 10856, 10996, 11132, 10928, 10896, 11062, 10846] dup_rows_per_dst=[4744, 5508, 5510, 6028, 5482, 5414, 5714, 5420] combine_rows_per_src=[10536, 10856, 10996, 11132, 10928, 10896, 11062, 10846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87430 payload_rows_remote=76304 dup_rows=43642 dup_rows_remote=38056 dedup_ratio=0.332962 router_tokens_per_expert=[495, 486, 516, 538, 474, 523, 474, 513, 537, 474, 494, 528, 516, 509, 525, 542, 531, 504, 516, 540, 484, 483, 511, 499, 507, 526, 484, 500, 516, 501, 513, 501, 482, 487, 490, 479, 479, 535, 485, 541, 472, 525, 502, 517, 510, 531, 487, 507, 485, 524, 512, 555, 489, 479, 506, 551, 495, 508, 501, 542, 492, 528, 538, 529, 490, 559, 517, 538, 505, 501, 529, 499, 494, 508, 533, 538, 550, 527, 534, 531, 485, 518, 536, 466, 487, 522, 498, 500, 494, 531, 531, 530, 526, 542, 516, 541, 517, 521, 522, 520, 493, 513, 547, 505, 509, 480, 540, 515, 491, 509, 504, 512, 535, 480, 549, 532, 532, 492, 499, 482, 499, 506, 549, 522, 500, 516, 534, 512] payload_rows_per_dst=[11126, 10896, 10816, 10916, 10960, 11144, 10746, 10826] dup_rows_per_dst=[5586, 5360, 5414, 5552, 5582, 5608, 5220, 5320] combine_rows_per_src=[11126, 10896, 10816, 10916, 10960, 11144, 10746, 10826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87340 payload_rows_remote=76258 dup_rows=43732 dup_rows_remote=38254 dedup_ratio=0.333649 router_tokens_per_expert=[504, 505, 541, 520, 538, 514, 502, 488, 508, 510, 500, 545, 544, 511, 518, 542, 526, 496, 507, 515, 510, 480, 514, 481, 521, 501, 519, 506, 503, 521, 505, 510, 520, 505, 465, 532, 478, 538, 536, 492, 525, 511, 502, 529, 546, 492, 577, 543, 495, 503, 532, 496, 486, 536, 545, 510, 510, 492, 486, 490, 473, 498, 538, 474, 492, 506, 489, 526, 514, 470, 554, 474, 487, 498, 465, 511, 498, 520, 514, 471, 522, 503, 470, 490, 532, 533, 552, 492, 502, 504, 528, 532, 549, 544, 496, 531, 543, 482, 506, 514, 515, 528, 500, 499, 522, 514, 540, 525, 539, 534, 511, 508, 509, 510, 514, 512, 507, 516, 526, 465, 491, 529, 533, 516, 493, 543, 531, 532] payload_rows_per_dst=[11008, 10804, 11014, 10782, 10726, 11082, 10952, 10972] dup_rows_per_dst=[5572, 5426, 5568, 5346, 5252, 5478, 5608, 5482] combine_rows_per_src=[11008, 10804, 11014, 10782, 10726, 11082, 10952, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625131520 dispatch_meta_bytes=458152 combine_payload_bytes=625131520 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625589672 ingress_bytes=776126464 link_bytes=776126464 token_bytes=1250721192 weight_movement_bytes=150994944 total_nvlink_bytes=1401716136 weight_to_token_ratio=0.120726 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.842376044e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87436 payload_rows_remote=76472 dup_rows=43636 dup_rows_remote=38046 dedup_ratio=0.332916 router_tokens_per_expert=[522, 509, 498, 517, 564, 548, 505, 513, 519, 507, 508, 491, 494, 498, 515, 529, 458, 544, 505, 492, 487, 488, 518, 537, 496, 494, 511, 497, 473, 476, 505, 499, 537, 532, 506, 527, 476, 485, 464, 546, 559, 511, 557, 522, 515, 506, 488, 546, 528, 518, 494, 515, 499, 515, 487, 524, 533, 521, 495, 500, 527, 484, 499, 513, 508, 516, 525, 462, 510, 486, 513, 489, 520, 518, 534, 521, 499, 538, 492, 512, 485, 491, 547, 514, 535, 468, 510, 525, 489, 500, 466, 530, 508, 528, 527, 508, 513, 522, 513, 506, 519, 545, 532, 508, 509, 508, 535, 544, 537, 514, 547, 571, 516, 508, 495, 526, 511, 539, 506, 473, 522, 491, 492, 528, 512, 533, 493, 548] payload_rows_per_dst=[10942, 10772, 10964, 10832, 10952, 10934, 11098, 10942] dup_rows_per_dst=[5532, 5188, 5590, 5472, 5334, 5328, 5748, 5444] combine_rows_per_src=[10942, 10772, 10964, 10832, 10952, 10934, 11098, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87364 payload_rows_remote=76458 dup_rows=43708 dup_rows_remote=38368 dedup_ratio=0.333466 router_tokens_per_expert=[520, 529, 519, 494, 515, 525, 514, 497, 494, 553, 508, 569, 504, 526, 506, 505, 505, 485, 520, 462, 509, 489, 513, 516, 507, 542, 512, 508, 529, 507, 521, 498, 495, 550, 492, 493, 536, 506, 496, 506, 513, 496, 537, 500, 504, 544, 502, 470, 506, 503, 502, 526, 491, 520, 528, 513, 518, 507, 495, 532, 501, 511, 543, 530, 488, 502, 519, 529, 527, 542, 512, 493, 486, 502, 526, 523, 512, 513, 490, 522, 501, 503, 533, 534, 510, 494, 526, 496, 501, 508, 549, 522, 513, 514, 528, 488, 519, 496, 502, 525, 547, 515, 494, 482, 509, 498, 499, 533, 526, 490, 492, 494, 524, 512, 488, 463, 526, 527, 487, 544, 539, 548, 496, 561, 513, 483, 468, 563] payload_rows_per_dst=[11044, 10906, 10804, 10874, 10954, 10934, 10826, 11022] dup_rows_per_dst=[5512, 5340, 5476, 5578, 5418, 5506, 5416, 5462] combine_rows_per_src=[11044, 10906, 10804, 10874, 10954, 10934, 10826, 11022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627671040 dispatch_meta_bytes=459072 combine_payload_bytes=627671040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628130112 ingress_bytes=703168512 link_bytes=703168512 token_bytes=1255801152 weight_movement_bytes=75497472 total_nvlink_bytes=1331298624 weight_to_token_ratio=0.060119 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917166517e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76424 dup_rows=43666 dup_rows_remote=38134 dedup_ratio=0.333145 router_tokens_per_expert=[518, 521, 485, 506, 512, 510, 494, 523, 528, 515, 505, 509, 514, 521, 488, 526, 540, 478, 517, 506, 497, 516, 494, 558, 501, 552, 512, 520, 507, 499, 544, 511, 554, 519, 509, 544, 502, 554, 557, 495, 505, 504, 507, 534, 517, 499, 535, 514, 525, 524, 510, 528, 537, 503, 516, 467, 502, 552, 517, 490, 556, 495, 506, 529, 510, 504, 535, 486, 514, 526, 474, 486, 533, 500, 517, 509, 499, 511, 485, 522, 485, 521, 510, 531, 536, 516, 509, 516, 507, 531, 498, 502, 495, 518, 489, 534, 539, 536, 561, 499, 500, 509, 487, 524, 508, 464, 498, 511, 482, 510, 519, 514, 495, 475, 502, 511, 524, 538, 475, 510, 528, 499, 501, 518, 516, 483, 510, 448] payload_rows_per_dst=[10804, 11136, 11008, 10982, 10858, 10956, 10890, 10772] dup_rows_per_dst=[5546, 5368, 5690, 5532, 5364, 5440, 5432, 5294] combine_rows_per_src=[10804, 11136, 11008, 10982, 10858, 10956, 10890, 10772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625917952 dispatch_meta_bytes=459224 combine_payload_bytes=625917952 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626377176 ingress_bytes=852410368 link_bytes=852410368 token_bytes=1252295128 weight_movement_bytes=226492416 total_nvlink_bytes=1478787544 weight_to_token_ratio=0.180862 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080976332e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625082368 dispatch_meta_bytes=457440 combine_payload_bytes=625082368 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625539808 ingress_bytes=776077312 link_bytes=776077312 token_bytes=1250622176 weight_movement_bytes=150994944 total_nvlink_bytes=1401617120 weight_to_token_ratio=0.120736 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.841752728e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=624705536 dispatch_meta_bytes=458048 combine_payload_bytes=624705536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625163584 ingress_bytes=700203008 link_bytes=700203008 token_bytes=1249869120 weight_movement_bytes=75497472 total_nvlink_bytes=1325366592 weight_to_token_ratio=0.060404 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.879559752e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626458624 dispatch_meta_bytes=458072 combine_payload_bytes=626458624 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626916696 ingress_bytes=852951040 link_bytes=852951040 token_bytes=1253375320 weight_movement_bytes=226492416 total_nvlink_bytes=1479867736 weight_to_token_ratio=0.180706 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081661981e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626343936 dispatch_meta_bytes=459304 combine_payload_bytes=626343936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626803240 ingress_bytes=701841408 link_bytes=701841408 token_bytes=1253147176 weight_movement_bytes=75497472 total_nvlink_bytes=1328644648 weight_to_token_ratio=0.060246 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900336970e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=458232 combine_payload_bytes=626065408 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626523640 ingress_bytes=852557824 link_bytes=852557824 token_bytes=1252589048 weight_movement_bytes=226492416 total_nvlink_bytes=1479081464 weight_to_token_ratio=0.180819 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081163327e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87416 payload_rows_remote=76446 dup_rows=43656 dup_rows_remote=38098 dedup_ratio=0.333069 router_tokens_per_expert=[523, 480, 487, 514, 510, 534, 518, 521, 531, 519, 499, 549, 496, 527, 508, 533, 529, 528, 553, 520, 507, 503, 467, 542, 521, 484, 521, 542, 513, 528, 518, 515, 534, 531, 498, 533, 537, 500, 503, 489, 523, 486, 510, 514, 519, 493, 498, 510, 553, 520, 531, 526, 486, 519, 511, 483, 502, 487, 505, 497, 500, 503, 475, 499, 526, 520, 516, 499, 522, 528, 496, 514, 509, 484, 525, 525, 523, 523, 540, 514, 509, 503, 518, 488, 486, 517, 507, 512, 503, 560, 516, 488, 474, 497, 540, 522, 497, 495, 538, 501, 482, 542, 517, 489, 490, 530, 529, 504, 520, 530, 500, 544, 488, 485, 504, 510, 474, 526, 473, 521, 517, 525, 525, 523, 536, 510, 500, 492] payload_rows_per_dst=[11004, 11022, 10910, 10848, 10970, 10934, 10946, 10782] dup_rows_per_dst=[5494, 5560, 5446, 5346, 5558, 5346, 5470, 5436] combine_rows_per_src=[11004, 11022, 10910, 10848, 10970, 10934, 10946, 10782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87352 payload_rows_remote=76430 dup_rows=43720 dup_rows_remote=38200 dedup_ratio=0.333557 router_tokens_per_expert=[500, 501, 538, 496, 512, 520, 505, 515, 540, 482, 498, 504, 509, 507, 545, 480, 499, 479, 486, 530, 554, 523, 477, 502, 499, 539, 518, 523, 504, 516, 484, 514, 523, 492, 491, 494, 501, 542, 558, 522, 567, 484, 491, 478, 525, 525, 492, 537, 531, 509, 504, 542, 514, 518, 479, 531, 511, 522, 520, 509, 518, 478, 501, 472, 494, 568, 533, 475, 485, 567, 489, 502, 514, 519, 560, 526, 540, 492, 510, 503, 553, 520, 512, 547, 517, 496, 521, 452, 502, 491, 506, 514, 499, 460, 523, 522, 520, 498, 473, 519, 539, 525, 485, 534, 524, 508, 503, 538, 508, 512, 524, 511, 518, 482, 507, 534, 552, 494, 547, 460, 519, 528, 536, 518, 506, 506, 521, 495] payload_rows_per_dst=[10898, 10900, 11026, 10856, 10962, 10774, 10922, 11014] dup_rows_per_dst=[5406, 5394, 5418, 5462, 5592, 5496, 5520, 5432] combine_rows_per_src=[10898, 10900, 11026, 10856, 10962, 10774, 10922, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626245632 dispatch_meta_bytes=458176 combine_payload_bytes=626245632 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626703808 ingress_bytes=777240576 link_bytes=777240576 token_bytes=1252949440 weight_movement_bytes=150994944 total_nvlink_bytes=1403944384 weight_to_token_ratio=0.120512 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.856504553e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76404 dup_rows=43520 dup_rows_remote=37648 dedup_ratio=0.332031 router_tokens_per_expert=[501, 518, 531, 516, 508, 553, 480, 487, 531, 505, 549, 500, 467, 559, 555, 536, 494, 536, 535, 512, 505, 503, 506, 483, 504, 469, 531, 480, 474, 502, 510, 508, 543, 526, 507, 525, 505, 570, 499, 491, 448, 500, 503, 530, 537, 517, 502, 522, 512, 488, 549, 488, 513, 482, 550, 514, 491, 501, 517, 518, 530, 553, 490, 518, 471, 562, 527, 516, 549, 503, 520, 516, 518, 499, 507, 522, 473, 519, 486, 480, 475, 511, 517, 556, 529, 531, 520, 499, 526, 508, 448, 545, 517, 487, 530, 496, 524, 512, 542, 558, 505, 535, 507, 506, 466, 527, 530, 543, 492, 546, 502, 513, 448, 509, 465, 494, 513, 500, 517, 480, 511, 532, 519, 492, 517, 523, 534, 524] payload_rows_per_dst=[10646, 10886, 10978, 10990, 10894, 10940, 11070, 11148] dup_rows_per_dst=[5082, 5218, 5356, 5422, 5458, 5450, 5662, 5872] combine_rows_per_src=[10646, 10886, 10978, 10990, 10894, 10940, 11070, 11148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87016 payload_rows_remote=76200 dup_rows=44056 dup_rows_remote=38524 dedup_ratio=0.336121 router_tokens_per_expert=[540, 511, 505, 523, 502, 501, 519, 514, 549, 496, 485, 491, 539, 500, 526, 487, 522, 520, 497, 549, 544, 508, 473, 468, 491, 504, 501, 504, 534, 526, 495, 511, 493, 505, 513, 521, 486, 533, 487, 500, 530, 530, 524, 493, 477, 505, 517, 538, 512, 518, 492, 509, 560, 509, 481, 485, 514, 491, 489, 547, 512, 513, 510, 557, 531, 503, 519, 513, 512, 565, 549, 552, 513, 495, 534, 514, 517, 509, 505, 521, 532, 506, 531, 499, 483, 546, 523, 481, 495, 537, 482, 535, 520, 516, 492, 496, 554, 494, 551, 483, 494, 524, 507, 514, 558, 493, 512, 482, 491, 508, 501, 571, 503, 501, 516, 531, 538, 509, 509, 500, 483, 503, 513, 464, 466, 545, 486, 520] payload_rows_per_dst=[10884, 10832, 10866, 10904, 11008, 10816, 10900, 10806] dup_rows_per_dst=[5492, 5462, 5438, 5494, 5696, 5532, 5574, 5368] combine_rows_per_src=[10884, 10832, 10866, 10904, 11008, 10816, 10900, 10806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87298 payload_rows_remote=76354 dup_rows=43774 dup_rows_remote=37980 dedup_ratio=0.333969 router_tokens_per_expert=[492, 520, 509, 535, 541, 526, 523, 516, 500, 514, 475, 537, 523, 533, 525, 495, 465, 482, 553, 509, 504, 503, 489, 546, 531, 561, 513, 528, 496, 495, 468, 529, 550, 529, 535, 464, 497, 548, 531, 509, 490, 547, 549, 529, 539, 513, 525, 512, 504, 499, 501, 558, 470, 520, 504, 464, 487, 503, 473, 582, 516, 559, 531, 490, 520, 560, 497, 499, 496, 516, 537, 554, 516, 501, 495, 534, 513, 491, 476, 521, 466, 468, 516, 461, 488, 513, 539, 506, 476, 523, 509, 528, 507, 557, 491, 478, 493, 504, 545, 522, 521, 523, 507, 483, 498, 512, 491, 531, 519, 485, 519, 491, 502, 507, 528, 561, 518, 493, 508, 498, 441, 528, 495, 526, 527, 506, 508, 530] payload_rows_per_dst=[10944, 11314, 10750, 10718, 10940, 10988, 10738, 10906] dup_rows_per_dst=[5794, 5992, 5046, 5370, 5512, 5568, 5046, 5446] combine_rows_per_src=[10944, 11314, 10750, 10718, 10940, 10988, 10738, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87476 payload_rows_remote=76424 dup_rows=43596 dup_rows_remote=38046 dedup_ratio=0.332611 router_tokens_per_expert=[508, 534, 505, 488, 523, 498, 515, 483, 504, 487, 575, 487, 498, 483, 538, 533, 512, 499, 528, 538, 493, 485, 523, 501, 512, 490, 506, 525, 526, 529, 503, 517, 517, 522, 523, 520, 504, 473, 556, 524, 526, 511, 528, 495, 523, 564, 513, 502, 506, 548, 543, 574, 526, 533, 532, 530, 501, 515, 500, 553, 516, 527, 494, 509, 508, 514, 483, 480, 537, 486, 511, 529, 521, 513, 486, 506, 499, 491, 539, 528, 501, 496, 508, 534, 501, 519, 477, 509, 549, 511, 499, 497, 529, 530, 496, 476, 492, 504, 497, 489, 507, 492, 542, 489, 518, 483, 538, 492, 522, 524, 514, 521, 514, 481, 509, 495, 481, 480, 513, 482, 488, 529, 513, 501, 519, 545, 529, 516] payload_rows_per_dst=[10868, 10978, 11052, 11142, 10842, 10850, 10856, 10888] dup_rows_per_dst=[5450, 5396, 5550, 5672, 5420, 5414, 5392, 5302] combine_rows_per_src=[10868, 10978, 11052, 11142, 10842, 10850, 10856, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87432 payload_rows_remote=76436 dup_rows=43640 dup_rows_remote=37996 dedup_ratio=0.332947 router_tokens_per_expert=[499, 509, 485, 469, 547, 506, 512, 483, 507, 472, 461, 481, 516, 525, 518, 513, 527, 551, 533, 532, 542, 499, 518, 519, 552, 496, 480, 521, 545, 487, 529, 489, 520, 521, 482, 524, 535, 497, 555, 525, 472, 484, 484, 550, 523, 508, 494, 529, 551, 543, 503, 526, 519, 502, 482, 555, 509, 501, 471, 481, 520, 480, 517, 497, 496, 519, 514, 500, 524, 508, 510, 463, 529, 525, 504, 532, 477, 500, 532, 480, 501, 481, 507, 472, 518, 521, 502, 540, 488, 521, 504, 502, 524, 475, 533, 522, 520, 513, 526, 501, 584, 516, 545, 527, 533, 531, 497, 532, 483, 520, 552, 534, 496, 481, 541, 530, 529, 477, 512, 501, 522, 485, 535, 534, 510, 524, 542, 496] payload_rows_per_dst=[10862, 10996, 10936, 10894, 10882, 10806, 11132, 10924] dup_rows_per_dst=[5144, 5644, 5470, 5420, 5344, 5416, 5696, 5506] combine_rows_per_src=[10862, 10996, 10936, 10894, 10882, 10806, 11132, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87242 payload_rows_remote=76372 dup_rows=43830 dup_rows_remote=38332 dedup_ratio=0.334396 router_tokens_per_expert=[514, 526, 501, 486, 509, 529, 540, 501, 478, 504, 521, 491, 507, 533, 540, 478, 542, 512, 530, 506, 545, 518, 461, 490, 505, 516, 491, 508, 510, 498, 540, 522, 537, 542, 525, 550, 556, 485, 491, 533, 504, 500, 513, 501, 525, 518, 559, 507, 474, 519, 496, 510, 519, 492, 529, 508, 528, 500, 508, 509, 533, 531, 531, 497, 521, 535, 539, 504, 476, 469, 492, 509, 525, 496, 494, 501, 521, 475, 544, 517, 529, 525, 509, 517, 545, 534, 518, 534, 520, 506, 518, 498, 507, 512, 489, 521, 521, 527, 468, 518, 502, 461, 516, 539, 508, 475, 537, 521, 522, 507, 466, 505, 544, 511, 509, 498, 494, 497, 484, 512, 515, 545, 518, 490, 500, 538, 512, 494] payload_rows_per_dst=[10906, 10954, 11038, 10870, 10840, 10960, 10776, 10898] dup_rows_per_dst=[5410, 5434, 5654, 5498, 5396, 5604, 5410, 5424] combine_rows_per_src=[10906, 10954, 11038, 10870, 10840, 10960, 10776, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626114560 dispatch_meta_bytes=458520 combine_payload_bytes=626114560 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626573080 ingress_bytes=852606976 link_bytes=852606976 token_bytes=1252687640 weight_movement_bytes=226492416 total_nvlink_bytes=1479180056 weight_to_token_ratio=0.180805 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081225659e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625901568 dispatch_meta_bytes=456208 combine_payload_bytes=625901568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626357776 ingress_bytes=701399040 link_bytes=701399040 token_bytes=1252259344 weight_movement_bytes=75497472 total_nvlink_bytes=1327756816 weight_to_token_ratio=0.060289 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894727121e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624230400 dispatch_meta_bytes=458896 combine_payload_bytes=624230400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624689296 ingress_bytes=699727872 link_bytes=699727872 token_bytes=1248919696 weight_movement_bytes=75497472 total_nvlink_bytes=1324417168 weight_to_token_ratio=0.060450 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.873534358e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625491968 dispatch_meta_bytes=457336 combine_payload_bytes=625491968 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625949304 ingress_bytes=776486912 link_bytes=776486912 token_bytes=1251441272 weight_movement_bytes=150994944 total_nvlink_bytes=1402436216 weight_to_token_ratio=0.120657 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.846947032e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=457880 combine_payload_bytes=626065408 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=626523288 ingress_bytes=928055296 link_bytes=928055296 token_bytes=1252588696 weight_movement_bytes=301989888 total_nvlink_bytes=1554578584 weight_to_token_ratio=0.241093 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176904749e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626163712 dispatch_meta_bytes=457728 combine_payload_bytes=626163712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626621440 ingress_bytes=701661184 link_bytes=701661184 token_bytes=1252785152 weight_movement_bytes=75497472 total_nvlink_bytes=1328282624 weight_to_token_ratio=0.060264 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898051476e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625639424 dispatch_meta_bytes=458816 combine_payload_bytes=625639424 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626098240 ingress_bytes=852131840 link_bytes=852131840 token_bytes=1251737664 weight_movement_bytes=226492416 total_nvlink_bytes=1478230080 weight_to_token_ratio=0.180942 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080623120e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87314 payload_rows_remote=76396 dup_rows=43758 dup_rows_remote=38278 dedup_ratio=0.333847 router_tokens_per_expert=[457, 505, 501, 484, 499, 496, 527, 484, 516, 532, 541, 517, 525, 531, 485, 553, 505, 496, 504, 508, 538, 483, 476, 500, 539, 514, 476, 508, 480, 553, 523, 513, 507, 491, 504, 508, 545, 510, 502, 517, 517, 523, 523, 490, 482, 543, 574, 511, 551, 504, 477, 542, 555, 492, 516, 501, 505, 514, 527, 536, 499, 556, 500, 503, 500, 510, 536, 527, 519, 533, 523, 514, 516, 488, 515, 527, 509, 508, 481, 493, 540, 503, 511, 518, 505, 528, 496, 500, 502, 483, 509, 498, 509, 479, 534, 502, 531, 512, 507, 468, 513, 521, 532, 498, 538, 483, 484, 535, 499, 509, 561, 513, 498, 532, 510, 501, 531, 518, 522, 554, 544, 471, 508, 484, 510, 537, 516, 486] payload_rows_per_dst=[10842, 10864, 10964, 10976, 10918, 10844, 10876, 11030] dup_rows_per_dst=[5464, 5368, 5530, 5580, 5480, 5390, 5532, 5414] combine_rows_per_src=[10842, 10864, 10964, 10976, 10918, 10844, 10876, 11030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87920 payload_rows_remote=76870 dup_rows=43152 dup_rows_remote=37938 dedup_ratio=0.329224 router_tokens_per_expert=[514, 514, 533, 561, 526, 540, 530, 482, 549, 515, 521, 504, 507, 466, 518, 492, 511, 494, 490, 533, 497, 513, 518, 499, 555, 515, 493, 509, 531, 523, 537, 495, 543, 547, 520, 509, 534, 526, 477, 551, 516, 539, 490, 486, 514, 494, 532, 528, 488, 544, 533, 503, 506, 492, 484, 502, 539, 531, 489, 536, 521, 508, 510, 488, 510, 507, 487, 537, 530, 519, 485, 485, 512, 495, 498, 504, 520, 549, 494, 511, 493, 557, 511, 532, 474, 533, 507, 495, 519, 548, 497, 531, 512, 499, 509, 506, 522, 469, 486, 533, 483, 503, 514, 530, 536, 491, 487, 513, 513, 531, 518, 503, 535, 478, 495, 528, 525, 489, 482, 519, 517, 517, 494, 490, 503, 525, 473, 503] payload_rows_per_dst=[11054, 10980, 11160, 10958, 10934, 10954, 11050, 10830] dup_rows_per_dst=[5490, 5446, 5452, 5390, 5352, 5492, 5214, 5316] combine_rows_per_src=[11054, 10980, 11160, 10958, 10934, 10954, 11050, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87578 payload_rows_remote=76756 dup_rows=43494 dup_rows_remote=38120 dedup_ratio=0.331833 router_tokens_per_expert=[559, 515, 574, 507, 494, 516, 537, 470, 524, 498, 496, 487, 507, 466, 550, 519, 512, 504, 502, 501, 542, 492, 493, 480, 505, 510, 526, 550, 506, 499, 507, 506, 542, 543, 512, 563, 481, 535, 517, 508, 512, 541, 481, 534, 536, 535, 490, 510, 521, 519, 475, 510, 539, 513, 557, 470, 528, 520, 504, 545, 530, 488, 476, 530, 481, 513, 523, 537, 527, 504, 494, 516, 532, 477, 505, 516, 507, 565, 476, 506, 502, 500, 496, 524, 478, 511, 532, 522, 501, 505, 533, 490, 497, 517, 535, 492, 507, 537, 511, 523, 495, 538, 506, 500, 544, 510, 502, 507, 510, 522, 500, 509, 524, 479, 487, 514, 467, 517, 515, 522, 522, 501, 502, 483, 493, 525, 532, 499] payload_rows_per_dst=[11000, 10816, 11090, 10682, 11282, 10862, 11024, 10822] dup_rows_per_dst=[5438, 5454, 5492, 5114, 5764, 5408, 5450, 5374] combine_rows_per_src=[11000, 10816, 11090, 10682, 11282, 10862, 11024, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458696 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626294728 ingress_bytes=701333504 link_bytes=701333504 token_bytes=1252130760 weight_movement_bytes=75497472 total_nvlink_bytes=1327628232 weight_to_token_ratio=0.060295 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893896032e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87496 payload_rows_remote=76622 dup_rows=43576 dup_rows_remote=38188 dedup_ratio=0.332458 router_tokens_per_expert=[523, 486, 558, 504, 500, 504, 476, 529, 553, 518, 518, 483, 479, 467, 483, 485, 515, 527, 480, 524, 530, 487, 457, 481, 520, 502, 505, 523, 508, 499, 493, 536, 526, 542, 524, 511, 511, 464, 514, 492, 527, 525, 526, 481, 535, 496, 495, 502, 515, 512, 517, 538, 542, 521, 514, 539, 556, 486, 533, 514, 486, 476, 522, 574, 503, 492, 536, 527, 521, 523, 532, 490, 527, 486, 472, 507, 507, 526, 511, 554, 502, 478, 515, 489, 504, 514, 576, 487, 486, 522, 499, 501, 509, 502, 535, 512, 498, 503, 476, 515, 524, 514, 537, 532, 500, 528, 512, 521, 504, 485, 529, 516, 498, 522, 513, 484, 494, 516, 527, 516, 565, 537, 522, 515, 534, 522, 557, 506] payload_rows_per_dst=[10860, 10836, 10914, 11040, 10960, 10874, 10936, 11076] dup_rows_per_dst=[5272, 5338, 5428, 5650, 5468, 5388, 5452, 5580] combine_rows_per_src=[10860, 10836, 10914, 11040, 10960, 10874, 10936, 11076] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87700 payload_rows_remote=76642 dup_rows=43372 dup_rows_remote=37758 dedup_ratio=0.330902 router_tokens_per_expert=[502, 485, 504, 546, 532, 537, 483, 482, 560, 497, 488, 531, 499, 544, 490, 507, 478, 493, 495, 524, 524, 520, 479, 509, 505, 486, 471, 493, 519, 517, 541, 529, 541, 469, 488, 572, 487, 538, 506, 589, 491, 495, 520, 533, 503, 506, 557, 541, 520, 475, 511, 519, 512, 508, 485, 568, 509, 518, 491, 488, 505, 545, 544, 487, 540, 537, 547, 500, 534, 500, 493, 508, 529, 515, 514, 530, 502, 503, 491, 497, 516, 514, 547, 505, 503, 503, 520, 502, 505, 523, 530, 546, 500, 472, 520, 525, 519, 527, 451, 567, 526, 497, 521, 526, 469, 500, 534, 520, 507, 509, 520, 503, 492, 497, 524, 507, 500, 496, 513, 471, 488, 490, 497, 495, 592, 510, 503, 503] payload_rows_per_dst=[11024, 10824, 11058, 10924, 11070, 10932, 10998, 10870] dup_rows_per_dst=[5350, 5342, 5614, 5446, 5410, 5530, 5394, 5286] combine_rows_per_src=[11024, 10824, 11058, 10924, 11070, 10932, 10998, 10870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87332 payload_rows_remote=76398 dup_rows=43740 dup_rows_remote=38102 dedup_ratio=0.333710 router_tokens_per_expert=[516, 505, 523, 542, 488, 509, 493, 474, 520, 517, 521, 504, 525, 513, 507, 492, 519, 494, 538, 486, 527, 518, 530, 506, 538, 521, 516, 496, 524, 521, 555, 497, 532, 463, 492, 529, 519, 555, 534, 469, 482, 465, 501, 499, 495, 468, 494, 522, 538, 478, 486, 527, 524, 540, 526, 532, 527, 489, 551, 504, 536, 513, 527, 495, 504, 505, 514, 543, 493, 537, 506, 514, 515, 520, 495, 524, 495, 505, 507, 518, 533, 490, 527, 523, 532, 489, 469, 505, 537, 510, 495, 486, 490, 534, 455, 508, 492, 504, 510, 519, 520, 524, 488, 478, 536, 532, 528, 507, 548, 541, 504, 539, 520, 505, 497, 499, 518, 485, 489, 487, 539, 498, 578, 502, 522, 532, 548, 522] payload_rows_per_dst=[10898, 10934, 10858, 10898, 10852, 10858, 11080, 10954] dup_rows_per_dst=[5400, 5638, 5180, 5688, 5538, 5308, 5460, 5528] combine_rows_per_src=[10898, 10934, 10858, 10898, 10852, 10858, 11080, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87238 payload_rows_remote=76330 dup_rows=43834 dup_rows_remote=38198 dedup_ratio=0.334427 router_tokens_per_expert=[508, 520, 558, 543, 556, 551, 502, 518, 522, 530, 575, 527, 529, 517, 520, 523, 488, 494, 501, 526, 491, 500, 460, 517, 518, 506, 525, 541, 506, 518, 494, 506, 524, 473, 537, 512, 496, 497, 507, 528, 527, 490, 548, 497, 521, 478, 498, 518, 502, 527, 531, 499, 527, 529, 537, 517, 509, 509, 499, 475, 545, 540, 506, 520, 486, 531, 517, 490, 512, 500, 467, 486, 502, 526, 532, 491, 510, 501, 456, 496, 503, 503, 469, 532, 529, 461, 512, 513, 536, 524, 507, 519, 498, 513, 536, 521, 525, 518, 509, 522, 533, 537, 516, 476, 520, 494, 486, 526, 517, 520, 526, 538, 489, 495, 489, 516, 493, 528, 518, 541, 543, 484, 485, 521, 476, 515, 495, 493] payload_rows_per_dst=[11232, 10832, 10836, 10908, 10748, 10962, 11004, 10716] dup_rows_per_dst=[5766, 5350, 5466, 5636, 5258, 5390, 5522, 5446] combine_rows_per_src=[11232, 10832, 10836, 10908, 10748, 10962, 11004, 10716] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=629719040 dispatch_meta_bytes=459232 combine_payload_bytes=629719040 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=630178272 ingress_bytes=780713984 link_bytes=780713984 token_bytes=1259897312 weight_movement_bytes=150994944 total_nvlink_bytes=1410892256 weight_to_token_ratio=0.119847 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.900552255e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87218 payload_rows_remote=76062 dup_rows=43854 dup_rows_remote=38032 dedup_ratio=0.334579 router_tokens_per_expert=[534, 488, 518, 502, 495, 537, 509, 551, 529, 512, 518, 501, 479, 512, 476, 515, 535, 500, 530, 518, 499, 530, 532, 516, 491, 505, 526, 519, 527, 532, 494, 489, 512, 488, 530, 525, 512, 504, 527, 531, 498, 533, 473, 466, 528, 525, 503, 517, 515, 471, 467, 494, 517, 515, 533, 519, 544, 501, 510, 498, 525, 538, 504, 480, 496, 521, 509, 475, 486, 500, 500, 492, 532, 473, 511, 507, 533, 468, 541, 511, 499, 517, 506, 471, 505, 517, 506, 480, 525, 479, 535, 541, 547, 504, 510, 533, 551, 501, 501, 505, 508, 510, 519, 470, 526, 534, 502, 534, 516, 550, 524, 520, 535, 573, 486, 525, 492, 452, 590, 516, 532, 515, 534, 546, 523, 505, 471, 518] payload_rows_per_dst=[11156, 11102, 10700, 10746, 10800, 11046, 10722, 10946] dup_rows_per_dst=[5822, 5578, 5018, 5322, 5310, 5908, 5216, 5680] combine_rows_per_src=[11156, 11102, 10700, 10746, 10800, 11046, 10722, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628785152 dispatch_meta_bytes=459504 combine_payload_bytes=628785152 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629244656 ingress_bytes=704282624 link_bytes=704282624 token_bytes=1258029808 weight_movement_bytes=75497472 total_nvlink_bytes=1333527280 weight_to_token_ratio=0.060012 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.931295025e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627687424 dispatch_meta_bytes=459240 combine_payload_bytes=627687424 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628146664 ingress_bytes=703184896 link_bytes=703184896 token_bytes=1255834088 weight_movement_bytes=75497472 total_nvlink_bytes=1331331560 weight_to_token_ratio=0.060117 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917374289e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627851264 dispatch_meta_bytes=457600 combine_payload_bytes=627851264 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628308864 ingress_bytes=854343680 link_bytes=854343680 token_bytes=1256160128 weight_movement_bytes=226492416 total_nvlink_bytes=1482652544 weight_to_token_ratio=0.180305 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083428044e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625852416 dispatch_meta_bytes=458000 combine_payload_bytes=625852416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626310416 ingress_bytes=701349888 link_bytes=701349888 token_bytes=1252162832 weight_movement_bytes=75497472 total_nvlink_bytes=1327660304 weight_to_token_ratio=0.060294 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894103804e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625295360 dispatch_meta_bytes=458112 combine_payload_bytes=625295360 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625753472 ingress_bytes=776290304 link_bytes=776290304 token_bytes=1251048832 weight_movement_bytes=150994944 total_nvlink_bytes=1402043776 weight_to_token_ratio=0.120695 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.844453766e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=623099904 dispatch_meta_bytes=456376 combine_payload_bytes=623099904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623556280 ingress_bytes=698597376 link_bytes=698597376 token_bytes=1246656184 weight_movement_bytes=75497472 total_nvlink_bytes=1322153656 weight_to_token_ratio=0.060560 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.859198078e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87590 payload_rows_remote=76604 dup_rows=43482 dup_rows_remote=38042 dedup_ratio=0.331741 router_tokens_per_expert=[508, 534, 508, 512, 505, 526, 528, 469, 501, 509, 551, 522, 500, 491, 526, 518, 484, 527, 484, 505, 520, 477, 503, 482, 532, 554, 512, 468, 519, 525, 492, 491, 523, 491, 537, 514, 502, 490, 525, 500, 505, 534, 526, 528, 489, 519, 533, 494, 493, 539, 518, 532, 503, 516, 491, 504, 545, 482, 543, 504, 522, 493, 532, 522, 496, 510, 504, 507, 544, 510, 543, 461, 520, 488, 518, 561, 497, 515, 500, 539, 523, 528, 517, 517, 476, 509, 539, 499, 517, 505, 472, 502, 474, 500, 545, 502, 508, 504, 519, 532, 510, 522, 490, 531, 488, 502, 498, 554, 525, 508, 477, 505, 542, 479, 513, 512, 495, 497, 519, 546, 538, 546, 524, 543, 466, 532, 531, 510] payload_rows_per_dst=[10972, 10820, 10940, 10986, 10986, 10880, 10966, 11040] dup_rows_per_dst=[5444, 5330, 5480, 5492, 5440, 5370, 5380, 5546] combine_rows_per_src=[10972, 10820, 10940, 10986, 10986, 10880, 10966, 11040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87532 payload_rows_remote=76424 dup_rows=43540 dup_rows_remote=37984 dedup_ratio=0.332184 router_tokens_per_expert=[502, 513, 531, 485, 490, 519, 535, 503, 550, 534, 506, 527, 525, 497, 506, 511, 497, 509, 511, 484, 534, 506, 464, 529, 539, 517, 547, 487, 522, 496, 523, 533, 519, 518, 587, 505, 505, 502, 529, 491, 497, 503, 523, 477, 504, 502, 482, 511, 516, 561, 506, 529, 494, 498, 495, 508, 494, 541, 526, 482, 550, 536, 516, 520, 513, 518, 532, 446, 525, 522, 517, 499, 484, 507, 521, 504, 520, 508, 524, 529, 481, 504, 533, 489, 525, 514, 506, 516, 494, 525, 544, 489, 461, 546, 499, 517, 542, 544, 542, 521, 528, 552, 476, 517, 534, 504, 503, 503, 533, 485, 487, 561, 517, 509, 479, 487, 505, 526, 492, 470, 535, 476, 523, 499, 490, 507, 538, 480] payload_rows_per_dst=[11026, 10896, 10822, 11024, 11028, 10878, 11108, 10750] dup_rows_per_dst=[5442, 5500, 5488, 5520, 5310, 5408, 5556, 5316] combine_rows_per_src=[11026, 10896, 10822, 11024, 11028, 10878, 11108, 10750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87190 payload_rows_remote=76044 dup_rows=43882 dup_rows_remote=38368 dedup_ratio=0.334793 router_tokens_per_expert=[467, 495, 497, 524, 526, 525, 524, 486, 495, 486, 527, 504, 525, 515, 515, 478, 535, 503, 540, 506, 491, 492, 521, 483, 475, 524, 521, 521, 495, 510, 547, 516, 541, 525, 511, 481, 501, 495, 510, 511, 509, 524, 496, 504, 476, 513, 510, 513, 479, 501, 471, 545, 493, 532, 534, 514, 494, 481, 509, 530, 541, 529, 546, 515, 514, 500, 476, 505, 512, 535, 554, 483, 496, 504, 511, 500, 494, 472, 496, 550, 488, 541, 459, 505, 510, 546, 508, 538, 499, 508, 540, 504, 509, 513, 516, 525, 503, 472, 500, 520, 549, 524, 510, 510, 504, 516, 502, 519, 542, 537, 542, 557, 551, 524, 516, 505, 512, 510, 521, 514, 514, 522, 511, 554, 492, 512, 532, 525] payload_rows_per_dst=[10490, 10882, 10672, 10710, 11126, 11192, 10972, 11146] dup_rows_per_dst=[5064, 5478, 5362, 5216, 5756, 5850, 5642, 5514] combine_rows_per_src=[10490, 10882, 10672, 10710, 11126, 11192, 10972, 11146] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=458584 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627998552 ingress_bytes=703037440 link_bytes=703037440 token_bytes=1255538520 weight_movement_bytes=75497472 total_nvlink_bytes=1331035992 weight_to_token_ratio=0.060132 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915504339e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87396 payload_rows_remote=76484 dup_rows=43676 dup_rows_remote=38282 dedup_ratio=0.333221 router_tokens_per_expert=[492, 503, 521, 531, 492, 486, 473, 509, 513, 499, 501, 501, 550, 515, 493, 548, 488, 501, 535, 504, 481, 513, 526, 488, 506, 483, 514, 557, 522, 493, 515, 523, 532, 502, 531, 524, 521, 515, 523, 517, 483, 540, 516, 519, 535, 520, 544, 511, 505, 508, 522, 486, 500, 513, 515, 552, 534, 505, 495, 514, 497, 513, 504, 517, 505, 509, 517, 510, 519, 509, 537, 500, 481, 498, 491, 512, 556, 496, 507, 498, 512, 505, 497, 527, 479, 509, 493, 521, 497, 484, 526, 512, 524, 496, 533, 538, 498, 526, 481, 513, 547, 543, 502, 516, 521, 519, 546, 533, 501, 485, 507, 487, 515, 531, 493, 544, 492, 524, 497, 469, 509, 542, 521, 545, 510, 512, 477, 543] payload_rows_per_dst=[10842, 10880, 11090, 10884, 10890, 10912, 10946, 10952] dup_rows_per_dst=[5412, 5418, 5576, 5476, 5400, 5394, 5504, 5496] combine_rows_per_src=[10842, 10880, 11090, 10884, 10890, 10912, 10946, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87312 payload_rows_remote=76326 dup_rows=43760 dup_rows_remote=38250 dedup_ratio=0.333862 router_tokens_per_expert=[543, 473, 503, 470, 510, 497, 480, 485, 502, 509, 509, 570, 496, 515, 530, 514, 508, 521, 505, 478, 475, 495, 514, 522, 521, 515, 530, 511, 512, 493, 517, 531, 531, 513, 517, 514, 551, 517, 514, 514, 492, 549, 509, 499, 503, 515, 484, 526, 477, 521, 492, 522, 528, 518, 508, 563, 548, 504, 546, 525, 504, 538, 512, 510, 529, 493, 532, 500, 532, 553, 502, 575, 501, 504, 491, 503, 527, 533, 538, 513, 466, 543, 495, 509, 532, 476, 516, 527, 476, 512, 494, 536, 498, 522, 499, 507, 510, 505, 500, 524, 521, 500, 505, 512, 487, 514, 548, 521, 516, 496, 487, 528, 522, 507, 486, 487, 525, 507, 493, 507, 485, 501, 504, 516, 518, 511, 533, 508] payload_rows_per_dst=[10902, 10852, 10986, 11032, 11030, 10798, 10898, 10814] dup_rows_per_dst=[5310, 5444, 5510, 5600, 5622, 5418, 5450, 5406] combine_rows_per_src=[10902, 10852, 10986, 11032, 11030, 10798, 10898, 10814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87904 payload_rows_remote=76758 dup_rows=43168 dup_rows_remote=37666 dedup_ratio=0.329346 router_tokens_per_expert=[501, 540, 534, 527, 516, 509, 529, 475, 479, 496, 503, 503, 508, 524, 527, 545, 458, 521, 491, 533, 516, 528, 513, 555, 536, 515, 528, 501, 503, 524, 501, 532, 514, 494, 492, 545, 504, 527, 510, 508, 549, 514, 510, 504, 542, 473, 500, 520, 549, 495, 533, 488, 501, 517, 533, 517, 566, 497, 494, 496, 526, 539, 542, 531, 499, 492, 505, 465, 536, 502, 484, 490, 508, 528, 480, 511, 504, 525, 506, 502, 531, 483, 516, 488, 505, 518, 498, 523, 534, 497, 502, 540, 494, 476, 529, 554, 532, 482, 494, 522, 480, 484, 497, 512, 536, 540, 513, 513, 535, 528, 514, 514, 491, 501, 495, 498, 502, 510, 536, 501, 490, 479, 502, 545, 528, 524, 485, 527] payload_rows_per_dst=[10970, 11102, 10980, 11146, 10842, 11012, 10994, 10858] dup_rows_per_dst=[5462, 5408, 5432, 5502, 5232, 5364, 5398, 5370] combine_rows_per_src=[10970, 11102, 10980, 11146, 10842, 11012, 10994, 10858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87232 payload_rows_remote=76404 dup_rows=43840 dup_rows_remote=38522 dedup_ratio=0.334473 router_tokens_per_expert=[540, 516, 504, 501, 510, 526, 511, 451, 512, 503, 497, 513, 490, 488, 518, 504, 533, 518, 501, 537, 498, 517, 506, 490, 477, 523, 453, 519, 477, 509, 537, 478, 487, 504, 505, 530, 534, 542, 503, 527, 509, 529, 530, 524, 495, 494, 513, 495, 518, 509, 494, 519, 515, 513, 503, 508, 503, 477, 512, 508, 495, 553, 523, 493, 502, 508, 519, 508, 508, 523, 502, 534, 503, 519, 526, 541, 472, 474, 543, 515, 486, 508, 510, 510, 496, 527, 523, 525, 500, 507, 543, 504, 516, 514, 483, 518, 554, 491, 504, 528, 496, 552, 527, 531, 545, 509, 524, 494, 519, 514, 513, 498, 549, 530, 484, 528, 541, 526, 521, 527, 534, 558, 513, 519, 505, 485, 524, 505] payload_rows_per_dst=[10828, 10828, 10932, 10840, 10878, 10830, 11012, 11084] dup_rows_per_dst=[5340, 5318, 5510, 5446, 5516, 5510, 5586, 5614] combine_rows_per_src=[10828, 10828, 10932, 10840, 10878, 10830, 11012, 11084] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=457632 combine_payload_bytes=626065408 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626523040 ingress_bytes=777060352 link_bytes=777060352 token_bytes=1252588448 weight_movement_bytes=150994944 total_nvlink_bytes=1403583392 weight_to_token_ratio=0.120546 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.854219059e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=622952448 dispatch_meta_bytes=457648 combine_payload_bytes=622952448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623410096 ingress_bytes=698449920 link_bytes=698449920 token_bytes=1246362544 weight_movement_bytes=75497472 total_nvlink_bytes=1321860016 weight_to_token_ratio=0.060574 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.857328128e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87422 payload_rows_remote=76450 dup_rows=43650 dup_rows_remote=37900 dedup_ratio=0.333023 router_tokens_per_expert=[485, 512, 505, 543, 525, 512, 521, 523, 481, 529, 568, 506, 517, 519, 477, 493, 472, 504, 499, 500, 520, 530, 491, 512, 552, 483, 489, 493, 554, 542, 498, 562, 546, 542, 517, 492, 539, 514, 499, 553, 514, 505, 514, 481, 516, 504, 534, 535, 505, 534, 493, 496, 568, 490, 499, 530, 509, 496, 541, 509, 505, 511, 525, 494, 479, 501, 531, 505, 490, 498, 539, 544, 501, 499, 537, 496, 509, 507, 494, 494, 496, 517, 495, 509, 503, 506, 526, 463, 466, 485, 506, 503, 561, 533, 518, 525, 486, 503, 495, 507, 477, 514, 551, 504, 499, 514, 541, 491, 512, 501, 488, 541, 530, 503, 518, 519, 508, 517, 540, 504, 558, 517, 485, 534, 495, 514, 487, 520] payload_rows_per_dst=[10972, 11010, 10980, 10978, 10774, 10926, 10732, 11050] dup_rows_per_dst=[5750, 5430, 5592, 5432, 5474, 5298, 5226, 5448] combine_rows_per_src=[10972, 11010, 10980, 10978, 10774, 10926, 10732, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=459064 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627015992 ingress_bytes=702054400 link_bytes=702054400 token_bytes=1253572920 weight_movement_bytes=75497472 total_nvlink_bytes=1329070392 weight_to_token_ratio=0.060226 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903038008e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625262592 dispatch_meta_bytes=458304 combine_payload_bytes=625262592 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625720896 ingress_bytes=700760064 link_bytes=700760064 token_bytes=1250983488 weight_movement_bytes=75497472 total_nvlink_bytes=1326480960 weight_to_token_ratio=0.060350 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.886624006e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628801536 dispatch_meta_bytes=457696 combine_payload_bytes=628801536 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629259232 ingress_bytes=779796480 link_bytes=779796480 token_bytes=1258060768 weight_movement_bytes=150994944 total_nvlink_bytes=1409055712 weight_to_token_ratio=0.120022 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.888917013e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625901568 dispatch_meta_bytes=459704 combine_payload_bytes=625901568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626361272 ingress_bytes=701399040 link_bytes=701399040 token_bytes=1252262840 weight_movement_bytes=75497472 total_nvlink_bytes=1327760312 weight_to_token_ratio=0.060289 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894727121e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87364 payload_rows_remote=76460 dup_rows=43708 dup_rows_remote=38094 dedup_ratio=0.333466 router_tokens_per_expert=[533, 497, 466, 473, 558, 565, 485, 523, 531, 558, 518, 494, 533, 527, 484, 525, 498, 525, 490, 535, 524, 548, 494, 521, 523, 517, 477, 515, 499, 501, 529, 510, 497, 511, 495, 519, 516, 539, 521, 535, 522, 513, 497, 491, 494, 534, 474, 532, 507, 512, 478, 560, 507, 504, 493, 528, 494, 523, 507, 523, 511, 493, 520, 511, 563, 519, 512, 494, 512, 494, 504, 542, 497, 477, 512, 496, 555, 511, 532, 539, 551, 511, 487, 511, 480, 518, 491, 522, 469, 543, 541, 484, 550, 492, 512, 483, 489, 507, 514, 497, 493, 515, 523, 534, 518, 506, 525, 507, 523, 520, 511, 504, 499, 518, 481, 475, 482, 527, 465, 523, 520, 506, 494, 497, 544, 508, 517, 553] payload_rows_per_dst=[11042, 10946, 10936, 10916, 10904, 10864, 10842, 10914] dup_rows_per_dst=[5498, 5466, 5444, 5426, 5614, 5426, 5530, 5304] combine_rows_per_src=[11042, 10946, 10936, 10916, 10904, 10864, 10842, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626278400 dispatch_meta_bytes=457400 combine_payload_bytes=626278400 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626735800 ingress_bytes=777273344 link_bytes=777273344 token_bytes=1253014200 weight_movement_bytes=150994944 total_nvlink_bytes=1404009144 weight_to_token_ratio=0.120505 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.856920097e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626360320 dispatch_meta_bytes=458216 combine_payload_bytes=626360320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626818536 ingress_bytes=701857792 link_bytes=701857792 token_bytes=1253178856 weight_movement_bytes=75497472 total_nvlink_bytes=1328676328 weight_to_token_ratio=0.060245 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900544742e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87626 payload_rows_remote=76378 dup_rows=43446 dup_rows_remote=37716 dedup_ratio=0.331467 router_tokens_per_expert=[542, 508, 495, 496, 501, 505, 552, 509, 484, 517, 511, 515, 531, 488, 505, 547, 506, 513, 514, 544, 484, 455, 473, 515, 464, 521, 507, 491, 495, 492, 507, 537, 493, 494, 529, 527, 521, 518, 517, 502, 495, 521, 504, 533, 455, 509, 514, 534, 523, 520, 508, 512, 482, 509, 538, 532, 451, 473, 511, 529, 532, 499, 536, 525, 541, 542, 544, 500, 502, 507, 503, 507, 491, 472, 500, 529, 521, 489, 519, 503, 527, 519, 559, 497, 545, 526, 488, 486, 545, 526, 522, 529, 525, 505, 505, 522, 539, 533, 498, 489, 471, 493, 546, 525, 488, 533, 517, 534, 507, 526, 518, 521, 495, 524, 521, 509, 527, 474, 506, 507, 541, 539, 500, 520, 509, 537, 500, 523] payload_rows_per_dst=[10960, 10804, 10878, 11024, 10642, 11064, 11006, 11248] dup_rows_per_dst=[5452, 5232, 5454, 5336, 5184, 5576, 5482, 5730] combine_rows_per_src=[10960, 10804, 10878, 11024, 10642, 11064, 11006, 11248] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87540 payload_rows_remote=76622 dup_rows=43532 dup_rows_remote=38230 dedup_ratio=0.332123 router_tokens_per_expert=[467, 497, 515, 509, 502, 529, 510, 510, 544, 525, 528, 481, 520, 533, 491, 505, 508, 521, 536, 497, 514, 505, 515, 513, 521, 526, 528, 553, 537, 552, 492, 503, 511, 518, 505, 527, 518, 557, 502, 501, 498, 517, 543, 486, 532, 485, 484, 520, 504, 542, 537, 508, 487, 511, 498, 536, 558, 498, 518, 512, 536, 516, 502, 504, 477, 511, 485, 518, 526, 511, 530, 481, 485, 501, 522, 522, 487, 513, 563, 497, 516, 482, 486, 494, 521, 524, 456, 517, 521, 532, 484, 522, 541, 511, 495, 513, 522, 528, 489, 526, 480, 520, 517, 484, 477, 485, 485, 505, 520, 513, 508, 551, 519, 493, 491, 511, 541, 458, 472, 534, 512, 520, 512, 554, 540, 538, 501, 528] payload_rows_per_dst=[10960, 11030, 10842, 11050, 10932, 10780, 10918, 11028] dup_rows_per_dst=[5372, 5612, 5566, 5484, 5326, 5450, 5302, 5420] combine_rows_per_src=[10960, 11030, 10842, 11050, 10932, 10780, 10918, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87570 payload_rows_remote=76462 dup_rows=43502 dup_rows_remote=38098 dedup_ratio=0.331894 router_tokens_per_expert=[513, 528, 534, 495, 505, 517, 491, 475, 472, 521, 536, 486, 517, 532, 474, 507, 501, 488, 545, 520, 531, 454, 510, 535, 532, 483, 502, 536, 484, 508, 504, 515, 549, 487, 495, 500, 510, 504, 504, 522, 545, 535, 531, 517, 515, 519, 500, 518, 523, 475, 554, 502, 522, 485, 533, 533, 494, 471, 494, 500, 502, 501, 498, 493, 531, 498, 521, 565, 522, 549, 533, 488, 489, 549, 519, 516, 514, 544, 511, 482, 484, 508, 518, 541, 529, 512, 517, 509, 505, 492, 503, 532, 501, 548, 518, 539, 485, 508, 506, 475, 521, 499, 530, 523, 516, 512, 527, 510, 496, 527, 514, 490, 541, 506, 552, 489, 462, 507, 553, 523, 507, 509, 487, 520, 511, 491, 529, 541] payload_rows_per_dst=[10858, 10864, 10972, 10900, 11016, 11108, 10930, 10922] dup_rows_per_dst=[5348, 5432, 5530, 5260, 5646, 5404, 5348, 5534] combine_rows_per_src=[10858, 10864, 10972, 10900, 11016, 11108, 10930, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76684 dup_rows=43598 dup_rows_remote=38304 dedup_ratio=0.332626 router_tokens_per_expert=[526, 519, 491, 512, 516, 490, 508, 524, 518, 518, 468, 541, 523, 530, 511, 514, 493, 520, 526, 509, 514, 496, 493, 523, 520, 465, 500, 502, 549, 549, 505, 536, 521, 491, 481, 514, 495, 450, 507, 469, 521, 530, 489, 524, 507, 495, 522, 526, 543, 490, 524, 491, 511, 495, 546, 537, 483, 514, 500, 493, 554, 557, 547, 561, 526, 541, 522, 513, 502, 501, 497, 516, 500, 518, 493, 534, 477, 501, 483, 505, 534, 477, 487, 520, 548, 487, 513, 487, 481, 506, 506, 575, 507, 553, 472, 529, 479, 500, 522, 466, 518, 522, 517, 531, 513, 522, 538, 559, 527, 494, 516, 513, 499, 460, 514, 523, 491, 525, 520, 495, 467, 548, 583, 524, 513, 535, 486, 508] payload_rows_per_dst=[10938, 11014, 10790, 11088, 10888, 10894, 11004, 10858] dup_rows_per_dst=[5480, 5386, 5294, 5604, 5370, 5470, 5470, 5524] combine_rows_per_src=[10938, 11014, 10790, 11088, 10888, 10894, 11004, 10858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87378 payload_rows_remote=76514 dup_rows=43694 dup_rows_remote=38300 dedup_ratio=0.333359 router_tokens_per_expert=[482, 522, 492, 527, 542, 528, 530, 533, 500, 501, 478, 548, 515, 494, 506, 553, 546, 526, 517, 525, 514, 497, 524, 472, 509, 530, 523, 569, 549, 506, 529, 502, 532, 525, 484, 493, 522, 540, 554, 519, 481, 574, 506, 471, 527, 484, 513, 523, 501, 532, 451, 498, 522, 536, 484, 495, 499, 523, 568, 502, 483, 514, 511, 510, 539, 504, 480, 485, 502, 474, 488, 528, 490, 480, 512, 525, 462, 507, 512, 501, 477, 529, 533, 484, 541, 515, 545, 503, 540, 481, 489, 506, 502, 504, 517, 510, 515, 528, 505, 531, 505, 535, 498, 503, 555, 533, 528, 498, 532, 476, 519, 510, 520, 525, 502, 518, 506, 558, 514, 491, 486, 478, 482, 489, 540, 513, 498, 514] payload_rows_per_dst=[10976, 10998, 11000, 10864, 10770, 10946, 10964, 10860] dup_rows_per_dst=[5526, 5678, 5496, 5394, 5208, 5406, 5578, 5408] combine_rows_per_src=[10976, 10998, 11000, 10864, 10770, 10946, 10964, 10860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87200 payload_rows_remote=76488 dup_rows=43872 dup_rows_remote=38372 dedup_ratio=0.334717 router_tokens_per_expert=[523, 470, 532, 480, 503, 530, 528, 500, 489, 512, 530, 496, 516, 503, 506, 524, 470, 497, 476, 500, 542, 504, 495, 503, 484, 534, 514, 529, 485, 522, 526, 525, 507, 508, 519, 499, 571, 517, 508, 518, 496, 488, 509, 524, 515, 511, 521, 544, 487, 493, 540, 515, 539, 548, 509, 505, 484, 501, 496, 520, 470, 482, 480, 509, 518, 516, 496, 548, 458, 541, 532, 492, 487, 552, 497, 531, 497, 536, 521, 506, 493, 538, 562, 511, 522, 530, 526, 497, 516, 524, 499, 520, 480, 537, 500, 549, 469, 484, 522, 492, 485, 549, 540, 476, 530, 497, 519, 499, 486, 529, 498, 559, 542, 521, 484, 513, 532, 502, 507, 518, 543, 523, 509, 540, 503, 484, 519, 549] payload_rows_per_dst=[10936, 10712, 11036, 10706, 10926, 10986, 10896, 11002] dup_rows_per_dst=[5348, 5500, 5474, 5450, 5530, 5622, 5372, 5576] combine_rows_per_src=[10936, 10712, 11036, 10706, 10926, 10986, 10896, 11002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87430 payload_rows_remote=76264 dup_rows=43642 dup_rows_remote=37888 dedup_ratio=0.332962 router_tokens_per_expert=[510, 569, 542, 530, 516, 481, 525, 493, 489, 540, 530, 505, 478, 522, 543, 534, 516, 513, 533, 510, 510, 509, 519, 487, 516, 523, 519, 489, 521, 508, 524, 520, 536, 520, 513, 544, 527, 515, 490, 482, 511, 522, 499, 489, 498, 498, 483, 521, 483, 528, 477, 494, 530, 478, 557, 541, 500, 522, 481, 473, 514, 517, 503, 522, 501, 515, 523, 493, 505, 522, 510, 515, 520, 469, 483, 492, 514, 522, 515, 546, 478, 496, 564, 495, 537, 477, 497, 474, 565, 501, 519, 542, 513, 507, 533, 516, 509, 539, 510, 514, 503, 495, 530, 522, 525, 513, 493, 498, 529, 535, 501, 517, 488, 522, 511, 491, 531, 494, 499, 507, 546, 477, 536, 530, 490, 535, 500, 495] payload_rows_per_dst=[11166, 11064, 10942, 10842, 10888, 10838, 10790, 10900] dup_rows_per_dst=[5754, 5680, 5402, 5360, 5504, 5274, 5264, 5404] combine_rows_per_src=[11166, 11064, 10942, 10842, 10888, 10838, 10790, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625688576 dispatch_meta_bytes=456376 combine_payload_bytes=625688576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626144952 ingress_bytes=701186048 link_bytes=701186048 token_bytes=1251833528 weight_movement_bytes=75497472 total_nvlink_bytes=1327331000 weight_to_token_ratio=0.060310 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892026083e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627687424 dispatch_meta_bytes=459408 combine_payload_bytes=627687424 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=628146832 ingress_bytes=703184896 link_bytes=703184896 token_bytes=1255834256 weight_movement_bytes=75497472 total_nvlink_bytes=1331331728 weight_to_token_ratio=0.060117 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917374289e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626376704 dispatch_meta_bytes=458240 combine_payload_bytes=626376704 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626834944 ingress_bytes=852869120 link_bytes=852869120 token_bytes=1253211648 weight_movement_bytes=226492416 total_nvlink_bytes=1479704064 weight_to_token_ratio=0.180730 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081558095e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628195328 dispatch_meta_bytes=459952 combine_payload_bytes=628195328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628655280 ingress_bytes=703692800 link_bytes=703692800 token_bytes=1256850608 weight_movement_bytes=75497472 total_nvlink_bytes=1332348080 weight_to_token_ratio=0.060069 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923815227e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87530 payload_rows_remote=76490 dup_rows=43542 dup_rows_remote=37988 dedup_ratio=0.332199 router_tokens_per_expert=[516, 492, 504, 495, 519, 496, 504, 500, 500, 478, 539, 529, 524, 498, 498, 533, 530, 510, 534, 532, 522, 507, 525, 527, 535, 504, 524, 543, 474, 478, 511, 497, 458, 507, 476, 526, 494, 503, 527, 534, 474, 501, 518, 556, 546, 501, 490, 484, 510, 515, 478, 534, 516, 502, 531, 496, 492, 525, 511, 492, 518, 529, 532, 496, 516, 519, 493, 527, 529, 514, 522, 537, 496, 479, 506, 548, 549, 505, 541, 516, 537, 489, 497, 535, 489, 522, 526, 516, 504, 510, 506, 519, 519, 491, 518, 514, 497, 597, 537, 497, 539, 493, 516, 500, 539, 526, 534, 515, 537, 518, 530, 528, 481, 499, 465, 515, 497, 525, 502, 510, 512, 490, 520, 492, 508, 499, 529, 450] payload_rows_per_dst=[10862, 11024, 10772, 10890, 11040, 11032, 11134, 10776] dup_rows_per_dst=[5388, 5482, 5418, 5464, 5554, 5352, 5672, 5212] combine_rows_per_src=[10862, 11024, 10772, 10890, 11040, 11032, 11134, 10776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626802688 dispatch_meta_bytes=459256 combine_payload_bytes=626802688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627261944 ingress_bytes=702300160 link_bytes=702300160 token_bytes=1254064632 weight_movement_bytes=75497472 total_nvlink_bytes=1329562104 weight_to_token_ratio=0.060202 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906154591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626589696 dispatch_meta_bytes=459440 combine_payload_bytes=626589696 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627049136 ingress_bytes=777584640 link_bytes=777584640 token_bytes=1253638832 weight_movement_bytes=150994944 total_nvlink_bytes=1404633776 weight_to_token_ratio=0.120445 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.860867768e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624754688 dispatch_meta_bytes=456608 combine_payload_bytes=624754688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625211296 ingress_bytes=700252160 link_bytes=700252160 token_bytes=1249965984 weight_movement_bytes=75497472 total_nvlink_bytes=1325463456 weight_to_token_ratio=0.060400 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.880183068e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626606080 dispatch_meta_bytes=457912 combine_payload_bytes=626606080 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627063992 ingress_bytes=777601024 link_bytes=777601024 token_bytes=1253670072 weight_movement_bytes=150994944 total_nvlink_bytes=1404665016 weight_to_token_ratio=0.120442 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861075541e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87766 payload_rows_remote=76654 dup_rows=43306 dup_rows_remote=37824 dedup_ratio=0.330399 router_tokens_per_expert=[496, 521, 504, 516, 511, 519, 548, 486, 520, 541, 496, 548, 481, 533, 538, 500, 502, 463, 499, 503, 508, 534, 501, 539, 518, 484, 505, 503, 478, 549, 508, 494, 542, 550, 544, 544, 488, 518, 517, 515, 515, 481, 478, 508, 498, 502, 537, 495, 528, 507, 472, 480, 507, 498, 560, 474, 544, 473, 494, 519, 504, 536, 472, 503, 551, 459, 540, 495, 515, 471, 505, 475, 510, 527, 516, 504, 515, 470, 506, 522, 546, 522, 521, 536, 556, 509, 513, 519, 508, 498, 504, 489, 492, 493, 495, 519, 523, 513, 520, 554, 496, 521, 546, 498, 515, 483, 533, 525, 526, 495, 514, 527, 499, 517, 504, 506, 528, 543, 538, 513, 482, 543, 520, 489, 527, 533, 545, 510] payload_rows_per_dst=[11006, 10958, 11012, 10838, 10772, 10966, 11102, 11112] dup_rows_per_dst=[5382, 5126, 5580, 5304, 5390, 5474, 5568, 5482] combine_rows_per_src=[11006, 10958, 11012, 10838, 10772, 10966, 11102, 11112] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87288 payload_rows_remote=76328 dup_rows=43784 dup_rows_remote=38218 dedup_ratio=0.334045 router_tokens_per_expert=[487, 489, 517, 481, 494, 472, 514, 481, 492, 515, 490, 523, 501, 538, 496, 508, 446, 535, 507, 481, 512, 543, 517, 516, 518, 515, 516, 511, 509, 516, 531, 489, 493, 489, 534, 550, 499, 481, 503, 505, 530, 534, 482, 537, 501, 483, 496, 512, 502, 506, 482, 513, 502, 523, 516, 477, 484, 468, 496, 516, 505, 552, 505, 516, 508, 523, 489, 552, 526, 517, 531, 542, 503, 572, 505, 534, 521, 500, 546, 528, 525, 516, 537, 542, 501, 559, 505, 494, 498, 508, 510, 502, 508, 556, 496, 506, 516, 517, 490, 547, 553, 498, 502, 515, 506, 494, 487, 556, 530, 531, 536, 494, 540, 522, 525, 534, 499, 521, 499, 514, 500, 505, 455, 524, 523, 510, 540, 541] payload_rows_per_dst=[10658, 10920, 10900, 10736, 11074, 10960, 11034, 11006] dup_rows_per_dst=[5338, 5404, 5358, 5390, 5720, 5566, 5510, 5498] combine_rows_per_src=[10658, 10920, 10900, 10736, 11074, 10960, 11034, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87472 payload_rows_remote=76604 dup_rows=43600 dup_rows_remote=38360 dedup_ratio=0.332642 router_tokens_per_expert=[511, 498, 517, 507, 469, 510, 499, 547, 484, 539, 496, 518, 553, 521, 540, 527, 519, 543, 545, 523, 512, 521, 503, 519, 554, 508, 519, 530, 510, 500, 492, 471, 512, 521, 499, 533, 502, 514, 526, 520, 478, 538, 494, 511, 500, 511, 490, 528, 519, 529, 506, 453, 513, 520, 505, 492, 514, 551, 519, 524, 494, 542, 528, 506, 528, 500, 489, 482, 508, 518, 537, 522, 498, 515, 505, 517, 492, 534, 533, 561, 460, 497, 509, 508, 557, 538, 515, 525, 515, 542, 520, 512, 515, 504, 535, 494, 502, 473, 498, 496, 525, 488, 546, 518, 513, 486, 503, 495, 511, 472, 517, 511, 509, 508, 531, 513, 503, 504, 473, 475, 514, 519, 539, 500, 488, 526, 510, 488] payload_rows_per_dst=[10900, 11020, 10938, 10922, 11038, 10958, 10868, 10828] dup_rows_per_dst=[5572, 5518, 5416, 5508, 5440, 5534, 5240, 5372] combine_rows_per_src=[10900, 11020, 10938, 10922, 11038, 10958, 10868, 10828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87210 payload_rows_remote=76396 dup_rows=43862 dup_rows_remote=38444 dedup_ratio=0.334641 router_tokens_per_expert=[515, 511, 548, 538, 495, 532, 516, 541, 520, 517, 502, 512, 516, 531, 533, 519, 473, 453, 518, 482, 519, 537, 538, 518, 529, 522, 525, 533, 502, 474, 528, 526, 504, 498, 494, 503, 516, 502, 524, 527, 501, 521, 480, 489, 503, 514, 533, 507, 537, 536, 483, 519, 509, 511, 513, 504, 513, 518, 516, 477, 499, 509, 498, 500, 484, 518, 490, 485, 509, 506, 475, 575, 505, 509, 508, 479, 519, 503, 540, 514, 530, 527, 485, 513, 550, 510, 531, 488, 540, 503, 503, 503, 509, 520, 531, 516, 515, 513, 544, 493, 520, 530, 534, 536, 468, 531, 526, 515, 499, 533, 431, 495, 507, 524, 495, 522, 512, 481, 537, 539, 510, 531, 489, 502, 521, 521, 483, 520] payload_rows_per_dst=[10918, 10912, 10814, 10928, 10874, 10974, 10870, 10920] dup_rows_per_dst=[5774, 5442, 5418, 5356, 5364, 5544, 5496, 5468] combine_rows_per_src=[10918, 10912, 10814, 10928, 10874, 10974, 10870, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76558 dup_rows=43648 dup_rows_remote=38378 dedup_ratio=0.333008 router_tokens_per_expert=[551, 552, 519, 494, 488, 525, 503, 535, 479, 474, 527, 527, 514, 523, 518, 494, 529, 539, 512, 543, 531, 547, 547, 508, 532, 566, 507, 498, 501, 484, 518, 543, 458, 501, 554, 533, 495, 510, 526, 475, 545, 522, 560, 507, 488, 508, 537, 536, 529, 521, 483, 475, 524, 531, 539, 489, 501, 523, 517, 515, 489, 472, 503, 457, 521, 503, 514, 509, 509, 543, 480, 495, 509, 533, 507, 516, 543, 518, 504, 490, 544, 480, 513, 482, 505, 525, 522, 516, 521, 508, 496, 535, 512, 506, 555, 525, 530, 455, 469, 547, 499, 490, 537, 476, 473, 483, 495, 510, 552, 473, 500, 526, 540, 527, 547, 530, 468, 493, 513, 498, 524, 522, 481, 496, 500, 485, 486, 521] payload_rows_per_dst=[10904, 11136, 10914, 10866, 10962, 10934, 10736, 10972] dup_rows_per_dst=[5542, 5674, 5596, 5270, 5426, 5556, 5294, 5290] combine_rows_per_src=[10904, 11136, 10914, 10866, 10962, 10934, 10736, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627949568 dispatch_meta_bytes=457912 combine_payload_bytes=627949568 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628407480 ingress_bytes=854441984 link_bytes=854441984 token_bytes=1256357048 weight_movement_bytes=226492416 total_nvlink_bytes=1482849464 weight_to_token_ratio=0.180277 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083552708e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87606 payload_rows_remote=76688 dup_rows=43466 dup_rows_remote=38082 dedup_ratio=0.331619 router_tokens_per_expert=[543, 502, 547, 539, 498, 494, 499, 542, 525, 548, 496, 486, 544, 544, 532, 511, 481, 520, 506, 507, 539, 515, 465, 539, 516, 479, 540, 519, 528, 443, 514, 540, 527, 515, 515, 519, 566, 501, 500, 525, 528, 497, 503, 505, 515, 522, 510, 476, 510, 490, 477, 472, 485, 494, 519, 541, 535, 501, 504, 469, 487, 502, 504, 520, 492, 503, 522, 521, 522, 524, 522, 541, 513, 527, 526, 563, 540, 499, 542, 568, 481, 480, 525, 495, 513, 509, 508, 507, 497, 519, 527, 488, 518, 507, 541, 500, 467, 520, 471, 474, 511, 527, 511, 510, 456, 509, 486, 506, 559, 479, 537, 514, 513, 497, 522, 508, 527, 538, 495, 501, 520, 485, 570, 489, 520, 548, 500, 491] payload_rows_per_dst=[11036, 10918, 10982, 10812, 11102, 10934, 10866, 10956] dup_rows_per_dst=[5664, 5384, 5466, 5208, 5748, 5296, 5208, 5492] combine_rows_per_src=[11036, 10918, 10982, 10812, 11102, 10934, 10866, 10956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87540 payload_rows_remote=76546 dup_rows=43532 dup_rows_remote=37866 dedup_ratio=0.332123 router_tokens_per_expert=[517, 555, 528, 516, 560, 563, 523, 507, 518, 504, 487, 518, 479, 492, 531, 526, 552, 474, 566, 524, 533, 523, 489, 507, 522, 519, 538, 536, 499, 514, 525, 497, 531, 510, 530, 491, 487, 511, 526, 497, 508, 510, 513, 551, 487, 501, 485, 484, 480, 458, 510, 491, 535, 506, 514, 503, 526, 471, 550, 555, 527, 482, 522, 489, 533, 529, 500, 510, 546, 511, 552, 510, 500, 516, 546, 515, 525, 471, 514, 521, 505, 527, 508, 489, 507, 495, 521, 539, 532, 515, 519, 490, 483, 496, 547, 509, 539, 488, 489, 545, 507, 507, 500, 505, 510, 516, 508, 475, 474, 486, 497, 545, 540, 492, 511, 511, 526, 490, 498, 497, 516, 509, 521, 521, 494, 465, 505, 485] payload_rows_per_dst=[10994, 11006, 10906, 11264, 11092, 10700, 10704, 10874] dup_rows_per_dst=[5666, 5640, 5338, 5772, 5638, 5174, 5170, 5134] combine_rows_per_src=[10994, 11006, 10906, 11264, 11092, 10700, 10704, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625278976 dispatch_meta_bytes=458184 combine_payload_bytes=625278976 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625737160 ingress_bytes=776273920 link_bytes=776273920 token_bytes=1251016136 weight_movement_bytes=150994944 total_nvlink_bytes=1402011080 weight_to_token_ratio=0.120698 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.844245994e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=459856 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627999824 ingress_bytes=854032384 link_bytes=854032384 token_bytes=1255539792 weight_movement_bytes=226492416 total_nvlink_bytes=1482032208 weight_to_token_ratio=0.180394 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083033277e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=459360 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626295392 ingress_bytes=776830976 link_bytes=776830976 token_bytes=1252131424 weight_movement_bytes=150994944 total_nvlink_bytes=1403126368 weight_to_token_ratio=0.120590 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851310248e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87550 payload_rows_remote=76600 dup_rows=43522 dup_rows_remote=38038 dedup_ratio=0.332047 router_tokens_per_expert=[530, 502, 509, 529, 520, 536, 538, 505, 506, 500, 510, 508, 534, 549, 501, 488, 483, 529, 504, 521, 522, 507, 527, 498, 512, 508, 489, 491, 508, 530, 535, 513, 534, 510, 518, 521, 518, 471, 522, 497, 534, 480, 559, 489, 503, 519, 485, 500, 511, 545, 519, 511, 501, 504, 490, 527, 483, 525, 509, 510, 523, 474, 541, 507, 495, 487, 513, 468, 496, 510, 500, 571, 502, 513, 541, 501, 536, 500, 555, 529, 505, 506, 460, 502, 503, 535, 517, 517, 524, 497, 485, 489, 522, 514, 515, 502, 540, 508, 534, 501, 491, 508, 515, 512, 512, 527, 537, 518, 507, 546, 504, 476, 525, 510, 515, 541, 549, 519, 523, 447, 509, 514, 530, 515, 461, 546, 513, 491] payload_rows_per_dst=[10992, 10916, 11012, 10902, 10950, 10876, 10954, 10948] dup_rows_per_dst=[5538, 5438, 5308, 5458, 5484, 5310, 5518, 5468] combine_rows_per_src=[10992, 10916, 11012, 10902, 10950, 10876, 10954, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627163136 dispatch_meta_bytes=459744 combine_payload_bytes=627163136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627622880 ingress_bytes=702660608 link_bytes=702660608 token_bytes=1254786016 weight_movement_bytes=75497472 total_nvlink_bytes=1330283488 weight_to_token_ratio=0.060168 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910725579e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628228096 dispatch_meta_bytes=459080 combine_payload_bytes=628228096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628687176 ingress_bytes=703725568 link_bytes=703725568 token_bytes=1256915272 weight_movement_bytes=75497472 total_nvlink_bytes=1332412744 weight_to_token_ratio=0.060066 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924230771e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627064832 dispatch_meta_bytes=457648 combine_payload_bytes=627064832 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627522480 ingress_bytes=778059776 link_bytes=778059776 token_bytes=1254587312 weight_movement_bytes=150994944 total_nvlink_bytes=1405582256 weight_to_token_ratio=0.120354 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866893162e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627507200 dispatch_meta_bytes=458552 combine_payload_bytes=627507200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627965752 ingress_bytes=703004672 link_bytes=703004672 token_bytes=1255472952 weight_movement_bytes=75497472 total_nvlink_bytes=1330970424 weight_to_token_ratio=0.060135 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915088795e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87428 payload_rows_remote=76516 dup_rows=43644 dup_rows_remote=38142 dedup_ratio=0.332977 router_tokens_per_expert=[482, 512, 513, 538, 489, 523, 496, 545, 499, 535, 532, 589, 508, 531, 516, 524, 537, 541, 528, 475, 526, 538, 498, 475, 487, 490, 514, 521, 519, 481, 544, 519, 497, 454, 559, 518, 512, 522, 478, 530, 489, 479, 478, 482, 550, 557, 542, 487, 504, 493, 523, 523, 513, 491, 538, 526, 508, 496, 529, 490, 520, 510, 473, 521, 529, 519, 503, 512, 505, 514, 500, 493, 493, 524, 538, 518, 515, 525, 514, 511, 526, 509, 529, 523, 517, 511, 521, 511, 553, 524, 521, 483, 514, 485, 510, 522, 496, 509, 511, 461, 529, 508, 558, 499, 510, 502, 506, 521, 479, 495, 521, 510, 491, 501, 478, 519, 501, 507, 507, 493, 538, 492, 503, 517, 545, 509, 512, 519] payload_rows_per_dst=[10618, 11010, 10800, 11310, 10878, 10680, 11220, 10912] dup_rows_per_dst=[5082, 5376, 5318, 5970, 5298, 5068, 6030, 5502] combine_rows_per_src=[10618, 11010, 10800, 11310, 10878, 10680, 11220, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87550 payload_rows_remote=76526 dup_rows=43522 dup_rows_remote=37962 dedup_ratio=0.332047 router_tokens_per_expert=[540, 514, 491, 526, 497, 506, 520, 494, 483, 485, 544, 513, 535, 489, 512, 562, 502, 536, 468, 502, 519, 558, 488, 511, 498, 515, 527, 533, 497, 512, 527, 475, 538, 542, 512, 518, 495, 511, 512, 499, 534, 509, 508, 562, 555, 524, 538, 515, 501, 516, 520, 554, 495, 497, 508, 491, 490, 513, 450, 499, 481, 510, 490, 513, 489, 495, 467, 514, 515, 497, 527, 479, 497, 543, 503, 507, 503, 483, 489, 496, 530, 511, 501, 537, 533, 514, 498, 513, 504, 523, 490, 494, 553, 543, 514, 534, 527, 525, 518, 505, 528, 492, 537, 529, 526, 470, 543, 481, 487, 511, 470, 562, 509, 530, 525, 514, 503, 540, 508, 541, 496, 540, 532, 509, 497, 508, 504, 494] payload_rows_per_dst=[10998, 11022, 11044, 10848, 10718, 11024, 10962, 10934] dup_rows_per_dst=[5424, 5314, 5700, 5208, 5290, 5560, 5460, 5566] combine_rows_per_src=[10998, 11022, 11044, 10848, 10718, 11024, 10962, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87638 payload_rows_remote=76784 dup_rows=43434 dup_rows_remote=38060 dedup_ratio=0.331375 router_tokens_per_expert=[533, 515, 527, 533, 504, 491, 512, 487, 490, 456, 498, 523, 525, 494, 523, 536, 519, 455, 533, 524, 519, 497, 523, 500, 514, 558, 469, 498, 551, 514, 502, 512, 528, 475, 546, 520, 523, 505, 542, 534, 513, 532, 534, 513, 524, 474, 522, 499, 520, 474, 522, 516, 504, 515, 513, 552, 476, 549, 502, 489, 479, 490, 518, 532, 475, 550, 488, 517, 506, 468, 491, 475, 524, 527, 498, 533, 533, 533, 508, 504, 558, 508, 508, 551, 521, 504, 517, 517, 553, 554, 511, 515, 509, 515, 514, 506, 476, 525, 483, 531, 496, 457, 510, 503, 511, 494, 520, 510, 490, 541, 545, 522, 510, 518, 522, 481, 504, 507, 537, 496, 500, 511, 522, 471, 543, 518, 537, 484] payload_rows_per_dst=[10888, 11038, 11054, 10880, 10906, 11076, 10854, 10942] dup_rows_per_dst=[5406, 5338, 5514, 5422, 5354, 5646, 5374, 5380] combine_rows_per_src=[10888, 11038, 11054, 10880, 10906, 11076, 10854, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87460 payload_rows_remote=76562 dup_rows=43612 dup_rows_remote=38026 dedup_ratio=0.332733 router_tokens_per_expert=[530, 532, 483, 503, 530, 517, 517, 495, 497, 478, 559, 488, 485, 534, 514, 508, 523, 498, 481, 531, 518, 540, 519, 515, 533, 489, 485, 500, 502, 518, 516, 533, 504, 487, 506, 507, 525, 518, 519, 547, 529, 522, 505, 532, 516, 512, 478, 535, 526, 496, 524, 510, 503, 519, 514, 532, 589, 471, 510, 512, 507, 512, 510, 506, 492, 530, 517, 474, 492, 543, 545, 535, 505, 531, 474, 518, 543, 555, 495, 508, 493, 479, 494, 520, 506, 496, 547, 499, 481, 521, 498, 502, 502, 504, 520, 516, 486, 527, 505, 540, 503, 539, 502, 506, 500, 497, 465, 558, 506, 493, 495, 515, 487, 523, 493, 523, 509, 558, 532, 506, 507, 482, 508, 526, 520, 513, 527, 496] payload_rows_per_dst=[11010, 10894, 10898, 11000, 11000, 10792, 10920, 10946] dup_rows_per_dst=[5330, 5508, 5586, 5482, 5514, 5364, 5354, 5474] combine_rows_per_src=[11010, 10894, 10898, 11000, 11000, 10792, 10920, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87408 payload_rows_remote=76406 dup_rows=43664 dup_rows_remote=37906 dedup_ratio=0.333130 router_tokens_per_expert=[545, 544, 527, 510, 524, 512, 473, 515, 522, 539, 509, 513, 461, 547, 538, 510, 537, 502, 550, 508, 499, 497, 491, 515, 546, 511, 483, 516, 481, 480, 486, 515, 471, 472, 515, 530, 501, 530, 549, 490, 477, 523, 499, 522, 530, 475, 514, 507, 559, 566, 521, 537, 510, 484, 537, 492, 536, 516, 495, 524, 518, 515, 541, 529, 530, 520, 517, 522, 521, 499, 491, 500, 535, 510, 547, 518, 504, 470, 505, 507, 497, 517, 523, 491, 531, 514, 531, 560, 508, 509, 533, 522, 532, 450, 516, 502, 462, 515, 519, 477, 513, 472, 530, 512, 522, 455, 546, 504, 520, 516, 521, 520, 476, 478, 525, 517, 510, 519, 486, 494, 483, 523, 551, 498, 533, 510, 505, 501] payload_rows_per_dst=[10894, 10884, 10878, 11002, 10882, 11014, 10904, 10950] dup_rows_per_dst=[5684, 5350, 5332, 5758, 5510, 5458, 5304, 5268] combine_rows_per_src=[10894, 10884, 10878, 11002, 10882, 11014, 10904, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87512 payload_rows_remote=76654 dup_rows=43560 dup_rows_remote=38184 dedup_ratio=0.332336 router_tokens_per_expert=[544, 545, 542, 507, 503, 536, 530, 517, 476, 531, 471, 515, 490, 511, 500, 501, 530, 491, 491, 525, 501, 524, 534, 502, 480, 504, 547, 463, 482, 510, 502, 531, 535, 500, 523, 504, 514, 526, 521, 489, 487, 523, 499, 500, 512, 514, 499, 538, 532, 552, 485, 495, 518, 510, 548, 535, 524, 538, 544, 477, 535, 530, 483, 563, 561, 473, 493, 477, 503, 464, 542, 499, 512, 524, 507, 532, 501, 487, 541, 544, 540, 518, 539, 532, 493, 493, 487, 482, 517, 546, 495, 483, 501, 521, 512, 460, 499, 488, 519, 537, 496, 503, 465, 489, 538, 528, 497, 558, 507, 514, 505, 489, 528, 543, 509, 461, 508, 550, 501, 514, 512, 483, 496, 540, 545, 483, 534, 529] payload_rows_per_dst=[10952, 10858, 10982, 11136, 10930, 10884, 10830, 10940] dup_rows_per_dst=[5486, 5376, 5386, 5602, 5390, 5354, 5434, 5532] combine_rows_per_src=[10952, 10858, 10982, 11136, 10930, 10884, 10830, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626819072 dispatch_meta_bytes=458632 combine_payload_bytes=626819072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627277704 ingress_bytes=702316544 link_bytes=702316544 token_bytes=1254096776 weight_movement_bytes=75497472 total_nvlink_bytes=1329594248 weight_to_token_ratio=0.060201 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906362363e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87496 payload_rows_remote=76452 dup_rows=43576 dup_rows_remote=37992 dedup_ratio=0.332458 router_tokens_per_expert=[526, 546, 513, 504, 500, 535, 526, 483, 506, 530, 492, 496, 543, 511, 487, 490, 509, 542, 514, 541, 510, 473, 519, 480, 501, 539, 493, 561, 519, 516, 481, 560, 525, 491, 508, 531, 455, 482, 485, 542, 518, 502, 533, 490, 473, 554, 519, 534, 548, 468, 512, 488, 509, 502, 482, 537, 496, 493, 533, 495, 495, 505, 535, 536, 516, 521, 512, 512, 544, 542, 524, 549, 516, 486, 543, 503, 488, 484, 517, 545, 515, 526, 533, 527, 460, 514, 507, 545, 536, 536, 528, 495, 528, 493, 549, 508, 489, 476, 498, 516, 490, 513, 473, 515, 497, 479, 484, 519, 522, 504, 526, 545, 475, 519, 520, 517, 502, 530, 545, 503, 472, 489, 514, 535, 489, 498, 531, 527] payload_rows_per_dst=[11044, 10982, 10938, 10924, 10922, 11006, 10792, 10888] dup_rows_per_dst=[5584, 5628, 5364, 5344, 5318, 5594, 5300, 5444] combine_rows_per_src=[11044, 10982, 10938, 10924, 10922, 11006, 10792, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629014528 dispatch_meta_bytes=459376 combine_payload_bytes=629014528 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629473904 ingress_bytes=780009472 link_bytes=780009472 token_bytes=1258488432 weight_movement_bytes=150994944 total_nvlink_bytes=1409483376 weight_to_token_ratio=0.119981 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.891618051e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626900992 dispatch_meta_bytes=457952 combine_payload_bytes=626900992 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627358944 ingress_bytes=777895936 link_bytes=777895936 token_bytes=1254259936 weight_movement_bytes=150994944 total_nvlink_bytes=1405254880 weight_to_token_ratio=0.120386 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864815440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625917952 dispatch_meta_bytes=457248 combine_payload_bytes=625917952 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626375200 ingress_bytes=852410368 link_bytes=852410368 token_bytes=1252293152 weight_movement_bytes=226492416 total_nvlink_bytes=1478785568 weight_to_token_ratio=0.180862 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080976332e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87700 payload_rows_remote=76700 dup_rows=43372 dup_rows_remote=37728 dedup_ratio=0.330902 router_tokens_per_expert=[506, 497, 506, 546, 510, 489, 503, 493, 497, 510, 528, 496, 484, 450, 501, 522, 496, 476, 515, 512, 509, 475, 510, 515, 511, 510, 535, 510, 492, 529, 503, 539, 554, 540, 494, 487, 508, 515, 500, 531, 487, 521, 498, 503, 549, 500, 522, 513, 518, 498, 501, 527, 481, 469, 525, 548, 536, 522, 545, 503, 514, 479, 528, 514, 524, 495, 553, 507, 510, 556, 520, 523, 537, 516, 497, 514, 509, 504, 518, 539, 532, 538, 519, 514, 515, 492, 475, 480, 499, 554, 539, 504, 517, 513, 516, 491, 505, 511, 487, 508, 510, 504, 503, 552, 509, 449, 509, 563, 483, 522, 519, 550, 510, 491, 520, 517, 515, 512, 510, 523, 522, 530, 511, 481, 512, 553, 506, 514] payload_rows_per_dst=[10726, 10958, 11026, 11026, 11000, 11002, 10996, 10966] dup_rows_per_dst=[5350, 5316, 5418, 5390, 5644, 5394, 5372, 5488] combine_rows_per_src=[10726, 10958, 11026, 11026, 11000, 11002, 10996, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=458352 combine_payload_bytes=627195904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627654256 ingress_bytes=702693376 link_bytes=702693376 token_bytes=1254850160 weight_movement_bytes=75497472 total_nvlink_bytes=1330347632 weight_to_token_ratio=0.060165 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911141123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627949568 dispatch_meta_bytes=459352 combine_payload_bytes=627949568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628408920 ingress_bytes=703447040 link_bytes=703447040 token_bytes=1256358488 weight_movement_bytes=75497472 total_nvlink_bytes=1331855960 weight_to_token_ratio=0.060092 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920698644e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626294784 dispatch_meta_bytes=457776 combine_payload_bytes=626294784 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626752560 ingress_bytes=777289728 link_bytes=777289728 token_bytes=1253047344 weight_movement_bytes=150994944 total_nvlink_bytes=1404042288 weight_to_token_ratio=0.120502 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.857127869e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628326400 dispatch_meta_bytes=457712 combine_payload_bytes=628326400 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=628784112 ingress_bytes=930316288 link_bytes=930316288 token_bytes=1257110512 weight_movement_bytes=301989888 total_nvlink_bytes=1559100400 weight_to_token_ratio=0.240225 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.179772005e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87458 payload_rows_remote=76484 dup_rows=43614 dup_rows_remote=38188 dedup_ratio=0.332748 router_tokens_per_expert=[499, 514, 489, 544, 469, 514, 545, 525, 483, 507, 522, 499, 509, 500, 521, 509, 498, 504, 499, 517, 485, 525, 479, 516, 510, 563, 543, 520, 484, 508, 507, 495, 548, 509, 557, 538, 509, 525, 515, 501, 540, 498, 505, 515, 503, 525, 497, 503, 529, 519, 502, 541, 498, 527, 523, 531, 494, 507, 539, 526, 504, 568, 492, 519, 508, 527, 488, 499, 527, 469, 480, 536, 481, 491, 505, 531, 505, 497, 546, 493, 483, 514, 535, 510, 566, 522, 533, 511, 539, 545, 487, 514, 548, 475, 492, 527, 475, 519, 497, 522, 500, 504, 505, 479, 551, 428, 540, 511, 503, 491, 546, 479, 540, 593, 469, 549, 511, 481, 519, 494, 481, 526, 536, 497, 489, 464, 516, 528] payload_rows_per_dst=[10632, 11006, 10970, 10724, 10872, 11094, 11186, 10974] dup_rows_per_dst=[4968, 5590, 5606, 5074, 5294, 5882, 5774, 5426] combine_rows_per_src=[10632, 11006, 10970, 10724, 10872, 11094, 11186, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87442 payload_rows_remote=76604 dup_rows=43630 dup_rows_remote=38296 dedup_ratio=0.332870 router_tokens_per_expert=[524, 504, 537, 506, 539, 499, 568, 549, 521, 500, 524, 528, 490, 507, 515, 516, 481, 528, 548, 547, 577, 552, 502, 481, 502, 553, 542, 510, 504, 536, 532, 542, 507, 510, 552, 510, 521, 484, 482, 538, 476, 491, 484, 501, 486, 526, 476, 462, 492, 510, 511, 541, 506, 476, 496, 482, 521, 547, 505, 499, 508, 501, 512, 514, 520, 488, 499, 536, 502, 485, 519, 491, 516, 523, 527, 502, 486, 515, 501, 555, 506, 474, 542, 528, 517, 519, 502, 514, 484, 486, 493, 509, 498, 460, 514, 540, 537, 495, 534, 515, 527, 490, 490, 509, 544, 480, 500, 504, 505, 524, 561, 522, 486, 526, 521, 541, 529, 500, 499, 524, 507, 493, 525, 484, 482, 500, 545, 495] payload_rows_per_dst=[11030, 11080, 10766, 10882, 10914, 10838, 11006, 10926] dup_rows_per_dst=[5624, 5794, 5246, 5360, 5416, 5334, 5468, 5388] combine_rows_per_src=[11030, 11080, 10766, 10882, 10914, 10838, 11006, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87456 payload_rows_remote=76494 dup_rows=43616 dup_rows_remote=38108 dedup_ratio=0.332764 router_tokens_per_expert=[556, 535, 501, 532, 554, 505, 490, 536, 517, 506, 528, 481, 527, 519, 501, 525, 479, 483, 519, 497, 497, 478, 542, 483, 526, 501, 483, 521, 457, 507, 497, 530, 516, 527, 574, 523, 502, 511, 515, 481, 488, 481, 484, 495, 528, 502, 494, 516, 544, 487, 524, 528, 539, 501, 454, 511, 552, 480, 525, 534, 493, 511, 493, 521, 522, 515, 491, 540, 502, 544, 501, 528, 507, 532, 511, 533, 520, 529, 483, 506, 493, 533, 529, 489, 528, 529, 519, 497, 545, 521, 496, 537, 516, 544, 503, 517, 509, 490, 498, 513, 562, 527, 491, 514, 507, 530, 510, 536, 492, 523, 481, 509, 502, 493, 545, 486, 519, 489, 502, 523, 520, 486, 513, 515, 500, 508, 525, 511] payload_rows_per_dst=[10946, 10838, 10894, 11064, 11010, 10928, 10962, 10814] dup_rows_per_dst=[5462, 5162, 5380, 5548, 5518, 5578, 5508, 5460] combine_rows_per_src=[10946, 10838, 10894, 11064, 11010, 10928, 10962, 10814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76484 dup_rows=43492 dup_rows_remote=37920 dedup_ratio=0.331818 router_tokens_per_expert=[491, 521, 555, 501, 480, 492, 533, 477, 503, 527, 506, 532, 526, 534, 488, 536, 498, 472, 509, 487, 543, 530, 514, 534, 506, 483, 536, 489, 492, 543, 502, 512, 474, 507, 529, 534, 523, 501, 504, 473, 522, 512, 515, 508, 491, 528, 513, 504, 535, 494, 492, 524, 508, 482, 524, 498, 510, 537, 508, 544, 557, 555, 533, 533, 510, 526, 500, 549, 510, 510, 488, 510, 517, 503, 504, 525, 525, 531, 495, 532, 498, 506, 517, 491, 464, 525, 509, 498, 525, 516, 524, 509, 494, 494, 505, 501, 519, 454, 545, 561, 526, 517, 543, 480, 505, 533, 519, 480, 509, 536, 510, 536, 533, 520, 494, 537, 480, 504, 528, 501, 502, 517, 506, 501, 490, 517, 495, 503] payload_rows_per_dst=[10944, 10894, 10856, 11096, 10980, 10812, 11058, 10940] dup_rows_per_dst=[5460, 5406, 5420, 5572, 5490, 5340, 5488, 5316] combine_rows_per_src=[10944, 10894, 10856, 11096, 10980, 10812, 11058, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87492 payload_rows_remote=76532 dup_rows=43580 dup_rows_remote=38178 dedup_ratio=0.332489 router_tokens_per_expert=[517, 521, 517, 524, 538, 521, 518, 453, 483, 540, 528, 527, 501, 544, 486, 492, 489, 513, 510, 473, 514, 538, 503, 500, 498, 496, 525, 520, 506, 541, 504, 500, 472, 469, 521, 544, 522, 474, 529, 489, 531, 474, 522, 489, 511, 573, 520, 541, 513, 539, 568, 545, 506, 533, 516, 510, 541, 522, 473, 514, 503, 518, 495, 501, 475, 500, 507, 552, 507, 509, 462, 526, 502, 506, 526, 475, 519, 502, 550, 493, 537, 504, 519, 509, 483, 489, 506, 510, 533, 510, 492, 489, 478, 510, 510, 502, 475, 522, 561, 531, 529, 576, 466, 512, 514, 529, 510, 529, 503, 495, 497, 530, 514, 514, 501, 495, 531, 515, 501, 517, 520, 540, 552, 520, 508, 506, 503, 510] payload_rows_per_dst=[10860, 10974, 10960, 11004, 10900, 10806, 11004, 10984] dup_rows_per_dst=[5560, 5286, 5402, 5590, 5322, 5356, 5554, 5510] combine_rows_per_src=[10860, 10974, 10960, 11004, 10900, 10806, 11004, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87436 payload_rows_remote=76544 dup_rows=43636 dup_rows_remote=38198 dedup_ratio=0.332916 router_tokens_per_expert=[530, 481, 515, 505, 500, 492, 515, 499, 495, 531, 501, 474, 526, 512, 516, 559, 476, 519, 514, 530, 523, 497, 510, 503, 524, 501, 511, 509, 498, 498, 531, 521, 530, 503, 553, 509, 520, 482, 534, 528, 526, 506, 504, 479, 522, 470, 499, 527, 533, 533, 544, 527, 495, 490, 514, 505, 539, 545, 514, 534, 550, 527, 509, 473, 485, 506, 537, 527, 497, 482, 506, 537, 500, 534, 535, 533, 465, 522, 491, 491, 502, 494, 532, 533, 489, 481, 499, 479, 514, 532, 538, 508, 468, 535, 486, 509, 531, 511, 474, 545, 551, 560, 535, 566, 511, 527, 491, 493, 505, 513, 504, 500, 466, 516, 501, 490, 515, 503, 501, 527, 505, 524, 516, 508, 504, 484, 525, 547] payload_rows_per_dst=[10896, 10892, 10938, 10970, 10852, 10912, 11054, 10922] dup_rows_per_dst=[5406, 5438, 5446, 5694, 5444, 5286, 5580, 5342] combine_rows_per_src=[10896, 10892, 10938, 10970, 10852, 10912, 11054, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=458688 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627015616 ingress_bytes=702054400 link_bytes=702054400 token_bytes=1253572544 weight_movement_bytes=75497472 total_nvlink_bytes=1329070016 weight_to_token_ratio=0.060226 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903038008e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87158 payload_rows_remote=76180 dup_rows=43914 dup_rows_remote=38404 dedup_ratio=0.335037 router_tokens_per_expert=[522, 530, 553, 505, 513, 507, 523, 492, 505, 493, 532, 496, 518, 521, 515, 517, 525, 549, 481, 501, 505, 520, 507, 552, 489, 484, 499, 483, 513, 483, 561, 518, 502, 495, 559, 523, 514, 501, 575, 508, 559, 499, 528, 501, 511, 500, 476, 490, 497, 495, 527, 524, 513, 511, 525, 496, 480, 517, 498, 499, 496, 535, 513, 527, 507, 476, 500, 499, 490, 499, 526, 511, 518, 516, 555, 490, 516, 481, 531, 469, 489, 520, 531, 518, 561, 500, 506, 520, 568, 478, 522, 515, 465, 509, 547, 512, 488, 511, 510, 512, 493, 543, 485, 514, 529, 506, 567, 502, 494, 526, 542, 511, 530, 497, 527, 517, 523, 489, 527, 535, 489, 533, 455, 540, 515, 498, 505, 472] payload_rows_per_dst=[10978, 10912, 10966, 10748, 10770, 11022, 10958, 10804] dup_rows_per_dst=[5510, 5428, 5570, 5504, 5394, 5500, 5508, 5500] combine_rows_per_src=[10978, 10912, 10966, 10748, 10770, 11022, 10958, 10804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=459600 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627999568 ingress_bytes=703037440 link_bytes=703037440 token_bytes=1255539536 weight_movement_bytes=75497472 total_nvlink_bytes=1331037008 weight_to_token_ratio=0.060131 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915504339e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87280 payload_rows_remote=76352 dup_rows=43792 dup_rows_remote=38198 dedup_ratio=0.334106 router_tokens_per_expert=[540, 527, 511, 495, 507, 526, 542, 534, 507, 544, 487, 479, 502, 507, 496, 520, 502, 477, 454, 501, 489, 526, 503, 532, 512, 477, 520, 559, 536, 537, 513, 495, 512, 512, 518, 517, 493, 517, 506, 511, 520, 487, 547, 471, 554, 520, 459, 502, 528, 495, 515, 505, 523, 489, 515, 511, 508, 497, 530, 547, 507, 535, 506, 519, 508, 524, 501, 539, 490, 502, 534, 528, 523, 495, 530, 512, 517, 479, 562, 517, 459, 503, 509, 503, 500, 484, 533, 508, 490, 498, 560, 490, 505, 522, 481, 485, 523, 486, 502, 533, 516, 520, 525, 532, 520, 467, 511, 512, 485, 514, 501, 548, 475, 526, 521, 508, 538, 562, 525, 518, 517, 514, 508, 487, 566, 510, 527, 515] payload_rows_per_dst=[10848, 10914, 10946, 10872, 10928, 10752, 10956, 11064] dup_rows_per_dst=[5600, 5352, 5346, 5588, 5594, 5308, 5434, 5570] combine_rows_per_src=[10848, 10914, 10946, 10872, 10928, 10752, 10956, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626638848 dispatch_meta_bytes=458408 combine_payload_bytes=626638848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627097256 ingress_bytes=702136320 link_bytes=702136320 token_bytes=1253736104 weight_movement_bytes=75497472 total_nvlink_bytes=1329233576 weight_to_token_ratio=0.060218 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904076869e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=457616 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627014544 ingress_bytes=777551872 link_bytes=777551872 token_bytes=1253571472 weight_movement_bytes=150994944 total_nvlink_bytes=1404566416 weight_to_token_ratio=0.120452 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.860452224e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626950144 dispatch_meta_bytes=458840 combine_payload_bytes=626950144 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627408984 ingress_bytes=777945088 link_bytes=777945088 token_bytes=1254359128 weight_movement_bytes=150994944 total_nvlink_bytes=1405354072 weight_to_token_ratio=0.120376 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.865438756e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627048448 dispatch_meta_bytes=458968 combine_payload_bytes=627048448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627507416 ingress_bytes=702545920 link_bytes=702545920 token_bytes=1254555864 weight_movement_bytes=75497472 total_nvlink_bytes=1330053336 weight_to_token_ratio=0.060179 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909271174e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=624066560 dispatch_meta_bytes=458336 combine_payload_bytes=624066560 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=624524896 ingress_bytes=1001553920 link_bytes=1001553920 token_bytes=1248591456 weight_movement_bytes=377487360 total_nvlink_bytes=1626078816 weight_to_token_ratio=0.302331 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.270111350e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625475584 dispatch_meta_bytes=458200 combine_payload_bytes=625475584 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=625933784 ingress_bytes=700973056 link_bytes=700973056 token_bytes=1251409368 weight_movement_bytes=75497472 total_nvlink_bytes=1326906840 weight_to_token_ratio=0.060330 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889325044e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87454 payload_rows_remote=76568 dup_rows=43618 dup_rows_remote=38212 dedup_ratio=0.332779 router_tokens_per_expert=[475, 504, 496, 483, 520, 523, 490, 547, 537, 484, 512, 498, 530, 453, 482, 463, 511, 506, 488, 553, 528, 475, 539, 499, 477, 526, 528, 518, 531, 520, 548, 503, 502, 508, 493, 520, 502, 522, 517, 558, 542, 526, 466, 515, 470, 562, 540, 538, 506, 562, 530, 499, 558, 528, 524, 537, 519, 495, 523, 486, 498, 509, 504, 505, 503, 540, 496, 521, 539, 519, 502, 504, 518, 486, 541, 537, 503, 546, 510, 477, 463, 528, 496, 524, 536, 524, 501, 518, 504, 533, 529, 527, 530, 495, 513, 489, 531, 512, 546, 499, 511, 505, 484, 535, 513, 499, 484, 484, 522, 522, 495, 489, 533, 523, 528, 502, 480, 520, 526, 517, 527, 491, 480, 509, 496, 488, 530, 492] payload_rows_per_dst=[10788, 10924, 10948, 11038, 11002, 10986, 10882, 10886] dup_rows_per_dst=[5206, 5576, 5602, 5528, 5482, 5446, 5372, 5406] combine_rows_per_src=[10788, 10924, 10948, 11038, 11002, 10986, 10882, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87684 payload_rows_remote=76548 dup_rows=43388 dup_rows_remote=37758 dedup_ratio=0.331024 router_tokens_per_expert=[484, 496, 525, 514, 490, 510, 490, 495, 505, 491, 533, 546, 535, 518, 503, 509, 519, 500, 525, 523, 471, 507, 520, 532, 522, 521, 502, 521, 546, 495, 487, 541, 496, 498, 509, 473, 517, 531, 501, 494, 519, 513, 453, 501, 538, 506, 516, 520, 505, 510, 483, 503, 490, 543, 505, 529, 533, 508, 524, 476, 486, 500, 504, 541, 519, 532, 534, 471, 515, 492, 497, 490, 491, 504, 525, 495, 499, 524, 496, 502, 533, 546, 510, 488, 500, 560, 510, 501, 544, 563, 519, 509, 563, 530, 481, 526, 522, 529, 504, 501, 548, 540, 545, 554, 518, 512, 493, 496, 500, 505, 514, 487, 487, 530, 552, 502, 491, 514, 519, 496, 504, 527, 535, 502, 514, 530, 490, 505] payload_rows_per_dst=[10846, 10966, 10910, 10964, 10954, 11136, 10916, 10992] dup_rows_per_dst=[5442, 5498, 5260, 5316, 5218, 5630, 5620, 5404] combine_rows_per_src=[10846, 10966, 10910, 10964, 10954, 11136, 10916, 10992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87366 payload_rows_remote=76678 dup_rows=43706 dup_rows_remote=38358 dedup_ratio=0.333450 router_tokens_per_expert=[524, 504, 541, 489, 514, 545, 492, 478, 477, 520, 549, 496, 542, 508, 554, 530, 493, 493, 483, 467, 514, 510, 491, 503, 531, 502, 523, 513, 527, 522, 523, 468, 492, 516, 504, 504, 531, 530, 513, 535, 515, 548, 493, 522, 481, 544, 523, 546, 493, 490, 520, 484, 531, 487, 492, 527, 530, 489, 479, 537, 526, 523, 530, 532, 524, 482, 529, 493, 531, 516, 558, 565, 479, 508, 507, 548, 527, 547, 497, 521, 499, 480, 504, 525, 517, 526, 494, 507, 498, 518, 498, 526, 531, 548, 558, 509, 508, 514, 487, 488, 513, 494, 502, 531, 516, 463, 493, 510, 475, 548, 470, 506, 491, 552, 497, 512, 501, 496, 527, 548, 529, 489, 498, 487, 510, 496, 526, 496] payload_rows_per_dst=[10982, 10840, 11028, 11006, 11064, 10912, 10688, 10846] dup_rows_per_dst=[5544, 5286, 5566, 5334, 5600, 5564, 5348, 5464] combine_rows_per_src=[10982, 10840, 11028, 11006, 11064, 10912, 10688, 10846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76430 dup_rows=43546 dup_rows_remote=38016 dedup_ratio=0.332230 router_tokens_per_expert=[512, 495, 499, 502, 510, 500, 508, 499, 533, 563, 513, 519, 502, 520, 490, 508, 508, 517, 473, 506, 483, 544, 504, 518, 478, 558, 510, 491, 498, 511, 499, 552, 497, 522, 511, 495, 523, 509, 522, 507, 511, 452, 504, 512, 547, 488, 546, 509, 526, 507, 535, 515, 496, 544, 539, 523, 517, 523, 520, 544, 517, 508, 475, 524, 499, 504, 511, 473, 480, 552, 539, 501, 537, 490, 554, 521, 536, 471, 512, 498, 477, 519, 476, 513, 531, 499, 533, 496, 504, 545, 512, 517, 511, 510, 501, 537, 530, 499, 500, 515, 487, 522, 528, 522, 527, 482, 514, 533, 512, 506, 519, 497, 493, 512, 494, 516, 506, 510, 504, 519, 524, 539, 515, 536, 530, 498, 498, 499] payload_rows_per_dst=[10980, 10840, 10916, 11096, 10956, 10976, 10880, 10882] dup_rows_per_dst=[5366, 5460, 5394, 5530, 5400, 5386, 5506, 5504] combine_rows_per_src=[10980, 10840, 10916, 11096, 10956, 10976, 10880, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627245056 dispatch_meta_bytes=459120 combine_payload_bytes=627245056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627704176 ingress_bytes=702742528 link_bytes=702742528 token_bytes=1254949232 weight_movement_bytes=75497472 total_nvlink_bytes=1330446704 weight_to_token_ratio=0.060160 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911764440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87624 payload_rows_remote=76624 dup_rows=43448 dup_rows_remote=37918 dedup_ratio=0.331482 router_tokens_per_expert=[530, 509, 497, 500, 538, 494, 549, 489, 504, 522, 500, 519, 554, 493, 536, 497, 531, 516, 494, 508, 506, 535, 538, 532, 523, 468, 515, 511, 555, 560, 515, 509, 502, 499, 511, 547, 462, 523, 519, 497, 560, 506, 502, 530, 526, 538, 541, 502, 513, 499, 534, 492, 503, 526, 512, 521, 522, 563, 499, 480, 494, 488, 489, 555, 522, 500, 514, 514, 504, 497, 514, 539, 515, 505, 484, 510, 521, 507, 506, 524, 535, 498, 499, 507, 497, 495, 497, 459, 517, 520, 520, 505, 483, 505, 474, 502, 525, 499, 522, 470, 490, 531, 504, 543, 548, 484, 522, 499, 539, 522, 493, 482, 504, 493, 490, 493, 526, 490, 527, 546, 554, 535, 501, 471, 519, 512, 503, 508] payload_rows_per_dst=[11038, 11016, 11000, 10950, 10882, 10790, 10926, 11022] dup_rows_per_dst=[5424, 5616, 5530, 5430, 5470, 5236, 5420, 5322] combine_rows_per_src=[11038, 11016, 11000, 10950, 10882, 10790, 10926, 11022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87730 payload_rows_remote=76790 dup_rows=43342 dup_rows_remote=37932 dedup_ratio=0.330673 router_tokens_per_expert=[514, 489, 500, 492, 510, 519, 538, 504, 489, 504, 509, 517, 503, 513, 492, 552, 503, 506, 511, 496, 542, 525, 495, 537, 514, 510, 489, 534, 466, 511, 527, 509, 520, 517, 503, 501, 504, 545, 532, 506, 470, 492, 507, 496, 476, 470, 493, 541, 533, 481, 494, 520, 466, 488, 538, 511, 477, 522, 488, 489, 484, 497, 542, 551, 523, 534, 537, 536, 489, 499, 529, 502, 508, 507, 511, 509, 475, 528, 513, 522, 533, 536, 504, 501, 485, 545, 525, 511, 569, 538, 513, 523, 527, 549, 469, 512, 476, 487, 502, 509, 529, 487, 521, 515, 542, 548, 500, 483, 488, 539, 521, 526, 500, 512, 501, 549, 528, 516, 503, 547, 526, 536, 531, 504, 542, 503, 509, 520] payload_rows_per_dst=[10952, 10940, 10814, 10876, 11034, 11132, 10942, 11040] dup_rows_per_dst=[5338, 5410, 5332, 5286, 5410, 5548, 5404, 5614] combine_rows_per_src=[10952, 10940, 10814, 10876, 11034, 11132, 10942, 11040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627081216 dispatch_meta_bytes=457224 combine_payload_bytes=627081216 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627538440 ingress_bytes=702578688 link_bytes=702578688 token_bytes=1254619656 weight_movement_bytes=75497472 total_nvlink_bytes=1330117128 weight_to_token_ratio=0.060176 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909686718e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87582 payload_rows_remote=76402 dup_rows=43490 dup_rows_remote=37672 dedup_ratio=0.331802 router_tokens_per_expert=[536, 509, 485, 486, 492, 537, 494, 475, 549, 540, 516, 504, 453, 525, 527, 511, 466, 497, 518, 497, 508, 542, 520, 515, 537, 481, 557, 511, 514, 494, 496, 552, 509, 531, 517, 524, 522, 504, 521, 506, 515, 493, 509, 516, 521, 506, 539, 498, 491, 514, 520, 504, 489, 491, 516, 530, 534, 514, 513, 489, 514, 511, 508, 543, 516, 527, 517, 505, 512, 522, 506, 508, 541, 515, 569, 548, 485, 517, 513, 519, 503, 493, 496, 516, 531, 526, 458, 515, 492, 572, 500, 517, 505, 530, 554, 514, 532, 495, 514, 494, 491, 487, 510, 517, 492, 499, 514, 527, 516, 530, 491, 504, 507, 503, 524, 503, 539, 467, 514, 513, 501, 509, 483, 493, 491, 519, 497, 562] payload_rows_per_dst=[11180, 11072, 11006, 10656, 11174, 10910, 10738, 10846] dup_rows_per_dst=[5818, 5490, 5304, 5092, 5688, 5534, 5160, 5404] combine_rows_per_src=[11180, 11072, 11006, 10656, 11174, 10910, 10738, 10846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76734 dup_rows=43554 dup_rows_remote=38232 dedup_ratio=0.332291 router_tokens_per_expert=[505, 499, 571, 515, 535, 505, 535, 532, 478, 465, 497, 519, 512, 499, 517, 521, 482, 505, 553, 503, 515, 516, 506, 492, 494, 520, 504, 481, 518, 503, 513, 532, 540, 519, 509, 503, 515, 491, 531, 525, 509, 487, 513, 528, 492, 510, 553, 554, 523, 501, 484, 486, 493, 530, 512, 473, 507, 547, 502, 499, 526, 499, 549, 480, 518, 489, 508, 487, 475, 490, 510, 541, 527, 502, 516, 526, 506, 495, 503, 460, 488, 518, 500, 513, 560, 516, 527, 506, 533, 492, 533, 500, 532, 504, 526, 523, 498, 494, 529, 494, 548, 554, 444, 518, 517, 520, 464, 528, 525, 525, 511, 509, 530, 545, 553, 507, 548, 485, 515, 497, 528, 512, 527, 525, 523, 504, 490, 513] payload_rows_per_dst=[10958, 10900, 11012, 10798, 10784, 10968, 10974, 11124] dup_rows_per_dst=[5452, 5374, 5546, 5424, 5322, 5574, 5382, 5480] combine_rows_per_src=[10958, 10900, 11012, 10798, 10784, 10968, 10974, 11124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628146176 dispatch_meta_bytes=460144 combine_payload_bytes=628146176 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628606320 ingress_bytes=703643648 link_bytes=703643648 token_bytes=1256752496 weight_movement_bytes=75497472 total_nvlink_bytes=1332249968 weight_to_token_ratio=0.060073 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923191910e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626114560 dispatch_meta_bytes=457784 combine_payload_bytes=626114560 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626572344 ingress_bytes=852606976 link_bytes=852606976 token_bytes=1252686904 weight_movement_bytes=226492416 total_nvlink_bytes=1479179320 weight_to_token_ratio=0.180805 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081225659e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627703808 dispatch_meta_bytes=458168 combine_payload_bytes=627703808 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628161976 ingress_bytes=778698752 link_bytes=778698752 token_bytes=1255865784 weight_movement_bytes=150994944 total_nvlink_bytes=1406860728 weight_to_token_ratio=0.120232 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874996277e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=629063680 dispatch_meta_bytes=458888 combine_payload_bytes=629063680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629522568 ingress_bytes=704561152 link_bytes=704561152 token_bytes=1258586248 weight_movement_bytes=75497472 total_nvlink_bytes=1334083720 weight_to_token_ratio=0.059986 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.934827152e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625885184 dispatch_meta_bytes=456296 combine_payload_bytes=625885184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626341480 ingress_bytes=701382656 link_bytes=701382656 token_bytes=1252226664 weight_movement_bytes=75497472 total_nvlink_bytes=1327724136 weight_to_token_ratio=0.060291 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894519349e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628604928 dispatch_meta_bytes=459864 combine_payload_bytes=628604928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629064792 ingress_bytes=704102400 link_bytes=704102400 token_bytes=1257669720 weight_movement_bytes=75497472 total_nvlink_bytes=1333167192 weight_to_token_ratio=0.060030 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.929009531e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15047881632 prefetch_bytes=830472192 reduce_bytes=905969664 weight_movement_bytes=1736441856 egress_bytes=7526691744 ingress_bytes=9257631744 link_bytes=9257631744 total_nvlink_bytes=16784323488 weight_to_token_ratio=0.115394 prefetch_share_of_ingress=0.089707 dedup_ratio=0.332645 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=12 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.783316801e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87578 payload_rows_remote=76528 dup_rows=43494 dup_rows_remote=38054 dedup_ratio=0.331833 router_tokens_per_expert=[503, 507, 484, 497, 530, 496, 507, 497, 538, 483, 486, 499, 528, 468, 497, 526, 534, 490, 513, 479, 496, 517, 544, 512, 505, 521, 484, 526, 496, 496, 523, 543, 519, 530, 530, 502, 525, 491, 526, 524, 522, 505, 540, 526, 524, 494, 500, 544, 498, 533, 555, 545, 480, 519, 538, 488, 544, 482, 530, 492, 498, 493, 493, 557, 504, 530, 521, 512, 523, 500, 508, 511, 554, 547, 511, 521, 545, 530, 514, 479, 532, 464, 518, 493, 494, 475, 508, 501, 519, 493, 556, 508, 548, 535, 531, 515, 470, 506, 512, 507, 508, 502, 528, 509, 531, 524, 538, 489, 514, 550, 499, 514, 505, 495, 493, 496, 467, 493, 521, 484, 507, 490, 505, 526, 548, 505, 510, 518] payload_rows_per_dst=[10766, 10962, 10916, 10886, 10884, 11114, 11000, 11050] dup_rows_per_dst=[5294, 5396, 5492, 5202, 5398, 5674, 5598, 5440] combine_rows_per_src=[10766, 10962, 10916, 10886, 10884, 11114, 11000, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15048144576 prefetch_bytes=679477248 reduce_bytes=1132462080 weight_movement_bytes=1811939328 egress_bytes=7526823616 ingress_bytes=9333260288 link_bytes=9333260288 total_nvlink_bytes=16860083904 weight_to_token_ratio=0.120409 prefetch_share_of_ingress=0.072802 dedup_ratio=0.332745 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=15 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.863239834e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15055686320 prefetch_bytes=830472192 reduce_bytes=226492416 weight_movement_bytes=1056964608 egress_bytes=7530597040 ingress_bytes=8582053888 link_bytes=8582053888 total_nvlink_bytes=16112650928 weight_to_token_ratio=0.070204 prefetch_share_of_ingress=0.096768 dedup_ratio=0.332921 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=3 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.069376954e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76602 dup_rows=43566 dup_rows_remote=38032 dedup_ratio=0.332382 router_tokens_per_expert=[538, 586, 523, 485, 511, 480, 524, 534, 508, 540, 540, 474, 496, 473, 499, 502, 484, 501, 525, 528, 543, 532, 473, 523, 490, 529, 516, 506, 484, 532, 541, 521, 555, 499, 529, 534, 522, 502, 522, 541, 514, 516, 468, 513, 555, 543, 463, 481, 483, 526, 506, 523, 535, 500, 479, 511, 520, 514, 530, 508, 530, 518, 511, 518, 524, 522, 552, 497, 490, 507, 509, 503, 500, 491, 500, 554, 536, 507, 501, 524, 498, 482, 525, 541, 525, 515, 450, 488, 507, 540, 529, 520, 541, 515, 557, 486, 521, 498, 495, 510, 508, 500, 495, 527, 519, 510, 486, 485, 515, 481, 527, 448, 491, 514, 535, 544, 490, 530, 520, 515, 493, 504, 518, 510, 518, 482, 518, 483] payload_rows_per_dst=[10948, 10902, 11052, 10950, 10970, 10904, 10862, 10918] dup_rows_per_dst=[5478, 5554, 5462, 5474, 5464, 5534, 5188, 5412] combine_rows_per_src=[10948, 10902, 11052, 10950, 10970, 10904, 10862, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=458328 combine_payload_bytes=626917376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627375704 ingress_bytes=702414848 link_bytes=702414848 token_bytes=1254293080 weight_movement_bytes=75497472 total_nvlink_bytes=1329790552 weight_to_token_ratio=0.060191 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907608996e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87378 payload_rows_remote=76486 dup_rows=43694 dup_rows_remote=38296 dedup_ratio=0.333359 router_tokens_per_expert=[523, 515, 491, 521, 501, 536, 491, 496, 522, 493, 497, 492, 545, 493, 508, 504, 491, 506, 518, 540, 505, 529, 525, 525, 517, 521, 525, 491, 474, 505, 529, 490, 503, 523, 532, 563, 523, 559, 488, 531, 556, 499, 518, 509, 476, 470, 498, 483, 514, 506, 508, 540, 511, 531, 550, 483, 484, 541, 506, 467, 530, 496, 524, 474, 519, 501, 518, 518, 499, 513, 515, 522, 517, 528, 521, 557, 509, 500, 466, 527, 501, 508, 513, 548, 498, 513, 476, 513, 525, 513, 537, 510, 495, 516, 492, 530, 549, 529, 496, 475, 509, 502, 479, 568, 494, 498, 547, 509, 509, 487, 517, 477, 499, 509, 543, 550, 481, 528, 515, 516, 485, 519, 521, 540, 493, 529, 515, 515] payload_rows_per_dst=[10886, 10928, 11018, 10934, 10930, 10812, 10892, 10978] dup_rows_per_dst=[5370, 5454, 5444, 5396, 5530, 5564, 5398, 5538] combine_rows_per_src=[10886, 10928, 11018, 10934, 10930, 10812, 10892, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627523584 dispatch_meta_bytes=458536 combine_payload_bytes=627523584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627982120 ingress_bytes=703021056 link_bytes=703021056 token_bytes=1255505704 weight_movement_bytes=75497472 total_nvlink_bytes=1331003176 weight_to_token_ratio=0.060133 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915296567e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87402 payload_rows_remote=76372 dup_rows=43670 dup_rows_remote=38068 dedup_ratio=0.333176 router_tokens_per_expert=[525, 532, 505, 484, 561, 538, 537, 501, 516, 540, 531, 477, 499, 530, 526, 514, 521, 490, 464, 524, 467, 510, 499, 499, 506, 493, 535, 554, 464, 505, 504, 489, 540, 508, 460, 536, 509, 478, 497, 494, 524, 515, 518, 521, 531, 556, 513, 517, 513, 557, 517, 488, 521, 537, 525, 494, 514, 558, 528, 538, 514, 519, 499, 504, 536, 498, 516, 485, 456, 508, 531, 527, 548, 522, 547, 520, 540, 490, 521, 500, 496, 510, 492, 487, 505, 543, 502, 545, 515, 493, 473, 504, 534, 506, 505, 526, 483, 529, 523, 534, 488, 527, 538, 524, 479, 491, 530, 499, 522, 505, 483, 516, 505, 530, 536, 512, 502, 494, 488, 497, 499, 500, 505, 514, 502, 486, 533, 498] payload_rows_per_dst=[11030, 10832, 11064, 10968, 10872, 10914, 10866, 10856] dup_rows_per_dst=[5602, 5266, 5472, 5634, 5516, 5358, 5476, 5346] combine_rows_per_src=[11030, 10832, 11064, 10968, 10872, 10914, 10866, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87330 payload_rows_remote=76362 dup_rows=43742 dup_rows_remote=38344 dedup_ratio=0.333725 router_tokens_per_expert=[537, 497, 550, 504, 502, 540, 524, 510, 519, 543, 502, 476, 490, 542, 507, 555, 504, 514, 511, 480, 496, 532, 498, 518, 525, 528, 519, 535, 500, 534, 502, 545, 479, 497, 546, 530, 499, 554, 508, 491, 503, 537, 507, 514, 474, 480, 491, 518, 521, 519, 519, 515, 528, 518, 496, 529, 495, 555, 513, 511, 498, 504, 481, 507, 493, 486, 515, 502, 537, 512, 509, 506, 514, 498, 509, 545, 528, 527, 501, 501, 503, 504, 492, 495, 528, 490, 546, 483, 496, 516, 537, 518, 480, 514, 492, 557, 510, 489, 520, 518, 496, 498, 520, 549, 491, 514, 564, 548, 493, 541, 516, 511, 503, 528, 485, 533, 486, 505, 486, 496, 492, 517, 513, 520, 473, 516, 514, 481] payload_rows_per_dst=[11048, 10914, 10824, 10950, 10968, 10934, 10958, 10734] dup_rows_per_dst=[5548, 5568, 5432, 5468, 5398, 5368, 5598, 5362] combine_rows_per_src=[11048, 10914, 10824, 10950, 10968, 10934, 10958, 10734] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15043453664 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=7524475616 ingress_bytes=8953430016 link_bytes=8953430016 total_nvlink_bytes=16477905632 weight_to_token_ratio=0.095354 prefetch_share_of_ingress=0.084322 dedup_ratio=0.332420 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=9 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041973 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.461841292e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626573312 dispatch_meta_bytes=459128 combine_payload_bytes=626573312 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627032440 ingress_bytes=853065728 link_bytes=853065728 token_bytes=1253605752 weight_movement_bytes=226492416 total_nvlink_bytes=1480098168 weight_to_token_ratio=0.180673 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081807421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15040247640 prefetch_bytes=754974720 reduce_bytes=754974720 weight_movement_bytes=1509949440 egress_bytes=7522875224 ingress_bytes=9027321856 link_bytes=9027321856 total_nvlink_bytes=16550197080 weight_to_token_ratio=0.100394 prefetch_share_of_ingress=0.083632 dedup_ratio=0.332981 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=10 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.539929004e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625557504 dispatch_meta_bytes=458824 combine_payload_bytes=625557504 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626016328 ingress_bytes=852049920 link_bytes=852049920 token_bytes=1251573832 weight_movement_bytes=226492416 total_nvlink_bytes=1478066248 weight_to_token_ratio=0.180966 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080519234e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625639424 dispatch_meta_bytes=457760 combine_payload_bytes=625639424 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=626097184 ingress_bytes=776634368 link_bytes=776634368 token_bytes=1251736608 weight_movement_bytes=150994944 total_nvlink_bytes=1402731552 weight_to_token_ratio=0.120628 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.848816982e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15042703632 prefetch_bytes=754974720 reduce_bytes=754974720 weight_movement_bytes=1509949440 egress_bytes=7524102416 ingress_bytes=9028550656 link_bytes=9028550656 total_nvlink_bytes=16552653072 weight_to_token_ratio=0.100378 prefetch_share_of_ingress=0.083621 dedup_ratio=0.332996 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=10 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042433 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.541227580e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15020012264 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=7512748776 ingress_bytes=9168207872 link_bytes=9168207872 total_nvlink_bytes=16680956648 weight_to_token_ratio=0.110582 prefetch_share_of_ingress=0.090582 dedup_ratio=0.332873 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.688815087e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87414 payload_rows_remote=76516 dup_rows=43658 dup_rows_remote=38108 dedup_ratio=0.333084 router_tokens_per_expert=[537, 516, 494, 477, 528, 540, 514, 532, 474, 487, 514, 517, 506, 502, 497, 495, 564, 486, 513, 534, 537, 504, 486, 500, 553, 535, 479, 514, 492, 475, 516, 510, 504, 498, 523, 515, 512, 519, 520, 508, 491, 502, 496, 492, 530, 536, 507, 533, 525, 494, 518, 509, 527, 510, 495, 503, 492, 502, 506, 526, 525, 534, 497, 523, 526, 509, 515, 513, 547, 493, 501, 523, 484, 524, 511, 507, 525, 522, 532, 493, 491, 535, 528, 501, 501, 535, 514, 516, 523, 497, 501, 483, 464, 529, 500, 484, 528, 482, 485, 513, 551, 494, 541, 525, 506, 472, 528, 519, 507, 518, 554, 501, 515, 518, 497, 490, 569, 519, 557, 527, 510, 502, 484, 528, 531, 501, 526, 511] payload_rows_per_dst=[10926, 10964, 10890, 10886, 10970, 10798, 10898, 11082] dup_rows_per_dst=[5334, 5432, 5482, 5486, 5480, 5406, 5550, 5488] combine_rows_per_src=[10926, 10964, 10890, 10886, 10970, 10798, 10898, 11082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=2 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626819072 dispatch_meta_bytes=458496 combine_payload_bytes=626819072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627277568 ingress_bytes=702316544 link_bytes=702316544 token_bytes=1254096640 weight_movement_bytes=75497472 total_nvlink_bytes=1329594112 weight_to_token_ratio=0.060201 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906362363e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=2 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15059225184 prefetch_bytes=754974720 reduce_bytes=905969664 weight_movement_bytes=1660944384 egress_bytes=7532366432 ingress_bytes=9187803136 link_bytes=9187803136 total_nvlink_bytes=16720169568 weight_to_token_ratio=0.110294 prefetch_share_of_ingress=0.082171 dedup_ratio=0.332563 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=12 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.709523048e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none + [2026-08-29 23:36:22] iteration 3/ 12 | consumed samples: 24 | elapsed time per iteration (ms): 356.2 | throughput per GPU (TFLOP/s/GPU): 507.7 | learning rate: 6.000000E-05 | global batch size: 8 | lm loss: 9.699880E+00 | loss scale: 1.0 | grad norm: 69.393 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87202 payload_rows_remote=76270 dup_rows=43870 dup_rows_remote=38426 dedup_ratio=0.334702 router_tokens_per_expert=[532, 529, 504, 478, 514, 495, 495, 532, 493, 513, 525, 486, 532, 507, 536, 490, 502, 500, 517, 482, 525, 527, 539, 530, 484, 510, 547, 503, 500, 493, 532, 497, 526, 514, 519, 494, 458, 535, 502, 513, 490, 505, 506, 539, 514, 535, 540, 522, 531, 525, 499, 508, 525, 502, 486, 547, 483, 547, 487, 456, 524, 488, 528, 504, 519, 510, 460, 522, 494, 522, 479, 562, 502, 535, 486, 509, 531, 496, 545, 553, 526, 508, 510, 544, 497, 505, 526, 520, 494, 480, 487, 520, 505, 520, 553, 471, 495, 540, 521, 510, 529, 525, 516, 524, 521, 498, 479, 520, 495, 541, 523, 475, 533, 541, 526, 528, 521, 520, 508, 509, 470, 523, 508, 478, 524, 556, 486, 501] payload_rows_per_dst=[10918, 10932, 10928, 10858, 10932, 10906, 10844, 10884] dup_rows_per_dst=[5404, 5444, 5496, 5422, 5518, 5426, 5580, 5580] combine_rows_per_src=[10918, 10932, 10928, 10858, 10932, 10906, 10844, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87696 payload_rows_remote=76664 dup_rows=43376 dup_rows_remote=37884 dedup_ratio=0.330933 router_tokens_per_expert=[496, 480, 537, 538, 515, 514, 546, 498, 490, 506, 481, 507, 505, 534, 512, 529, 503, 511, 493, 507, 512, 522, 482, 491, 471, 496, 528, 519, 528, 542, 511, 530, 496, 524, 535, 479, 507, 495, 492, 532, 520, 541, 538, 537, 544, 492, 523, 519, 517, 487, 497, 493, 503, 534, 529, 557, 528, 504, 524, 476, 534, 523, 487, 525, 531, 504, 521, 537, 495, 462, 506, 535, 493, 512, 506, 498, 491, 500, 469, 550, 557, 529, 512, 521, 491, 517, 499, 523, 544, 484, 511, 490, 526, 507, 509, 533, 528, 531, 507, 485, 514, 485, 496, 479, 526, 530, 505, 539, 477, 503, 535, 490, 497, 495, 502, 499, 519, 535, 484, 537, 542, 491, 506, 501, 532, 504, 535, 538] payload_rows_per_dst=[10964, 10932, 11046, 10922, 10826, 11066, 10908, 11032] dup_rows_per_dst=[5382, 5250, 5502, 5330, 5444, 5440, 5536, 5492] combine_rows_per_src=[10964, 10932, 11046, 10922, 10826, 11066, 10908, 11032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87454 payload_rows_remote=76578 dup_rows=43618 dup_rows_remote=38144 dedup_ratio=0.332779 router_tokens_per_expert=[568, 468, 501, 550, 512, 485, 489, 531, 496, 530, 519, 491, 507, 514, 491, 489, 464, 561, 523, 481, 503, 541, 488, 504, 501, 524, 511, 518, 534, 499, 522, 528, 501, 531, 477, 505, 511, 513, 532, 512, 501, 509, 507, 508, 502, 519, 497, 550, 500, 514, 507, 501, 553, 501, 542, 499, 531, 490, 503, 510, 502, 537, 561, 507, 506, 486, 492, 567, 523, 506, 533, 532, 512, 574, 490, 525, 508, 474, 500, 487, 499, 519, 510, 485, 507, 484, 506, 544, 505, 480, 512, 515, 494, 527, 502, 504, 504, 540, 555, 506, 543, 525, 505, 458, 523, 546, 514, 469, 543, 536, 503, 479, 504, 497, 533, 485, 551, 510, 508, 522, 492, 515, 481, 523, 509, 509, 553, 511] payload_rows_per_dst=[10892, 10930, 10876, 11032, 10894, 10846, 10970, 11014] dup_rows_per_dst=[5390, 5474, 5474, 5484, 5536, 5340, 5528, 5392] combine_rows_per_src=[10892, 10930, 10876, 11032, 10894, 10846, 10970, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76592 dup_rows=43506 dup_rows_remote=38104 dedup_ratio=0.331924 router_tokens_per_expert=[499, 509, 493, 501, 560, 508, 505, 478, 560, 542, 527, 522, 513, 532, 535, 517, 498, 491, 520, 530, 513, 551, 535, 516, 512, 513, 548, 523, 518, 485, 511, 509, 529, 517, 495, 501, 479, 513, 488, 529, 545, 470, 557, 493, 476, 504, 564, 499, 565, 553, 480, 457, 509, 543, 512, 509, 492, 509, 507, 524, 485, 552, 472, 484, 550, 497, 497, 514, 508, 505, 517, 486, 526, 518, 486, 481, 549, 502, 511, 498, 487, 550, 512, 545, 509, 485, 483, 499, 501, 524, 486, 493, 550, 490, 497, 533, 533, 498, 488, 531, 509, 487, 500, 489, 502, 513, 531, 531, 502, 510, 515, 549, 503, 497, 509, 542, 535, 496, 504, 486, 546, 528, 517, 523, 474, 501, 509, 503] payload_rows_per_dst=[10990, 10970, 10982, 10918, 10896, 10912, 10974, 10924] dup_rows_per_dst=[5612, 5576, 5336, 5388, 5394, 5376, 5402, 5422] combine_rows_per_src=[10990, 10970, 10982, 10918, 10896, 10912, 10974, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87554 payload_rows_remote=76480 dup_rows=43518 dup_rows_remote=37978 dedup_ratio=0.332016 router_tokens_per_expert=[530, 488, 514, 463, 503, 528, 503, 486, 547, 526, 501, 495, 528, 532, 464, 511, 470, 530, 555, 506, 501, 537, 495, 542, 507, 521, 521, 514, 510, 530, 491, 502, 502, 513, 508, 511, 492, 505, 511, 504, 502, 525, 511, 483, 516, 498, 511, 497, 526, 507, 515, 541, 502, 515, 522, 539, 506, 532, 549, 571, 475, 530, 489, 488, 505, 521, 529, 514, 516, 496, 485, 545, 499, 521, 495, 541, 486, 566, 509, 510, 515, 510, 553, 518, 568, 524, 509, 531, 508, 480, 512, 493, 513, 505, 529, 500, 513, 527, 499, 513, 511, 531, 499, 524, 559, 495, 476, 514, 499, 489, 495, 531, 553, 494, 455, 471, 483, 517, 495, 498, 506, 491, 549, 523, 480, 477, 519, 597] payload_rows_per_dst=[10888, 10908, 10942, 11074, 10958, 10988, 10896, 10900] dup_rows_per_dst=[5350, 5556, 5236, 5540, 5518, 5548, 5454, 5316] combine_rows_per_src=[10888, 10908, 10942, 11074, 10958, 10988, 10896, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87318 payload_rows_remote=76320 dup_rows=43754 dup_rows_remote=38298 dedup_ratio=0.333817 router_tokens_per_expert=[506, 498, 526, 520, 503, 478, 513, 507, 505, 530, 518, 460, 521, 562, 547, 461, 513, 487, 497, 470, 505, 561, 531, 538, 513, 488, 525, 508, 513, 535, 499, 516, 519, 509, 503, 525, 489, 486, 537, 487, 489, 501, 516, 498, 520, 490, 456, 517, 504, 534, 553, 480, 481, 536, 532, 498, 507, 500, 504, 521, 470, 506, 513, 509, 508, 540, 477, 486, 499, 548, 543, 518, 522, 497, 531, 503, 500, 528, 496, 527, 523, 495, 513, 509, 531, 531, 513, 484, 502, 537, 506, 481, 514, 557, 516, 515, 491, 502, 535, 552, 522, 539, 502, 537, 514, 492, 527, 511, 513, 501, 509, 481, 552, 496, 489, 519, 507, 505, 533, 571, 472, 516, 548, 452, 536, 569, 516, 533] payload_rows_per_dst=[10908, 10936, 10778, 10894, 10928, 10998, 10978, 10898] dup_rows_per_dst=[5402, 5462, 5306, 5402, 5518, 5456, 5478, 5730] combine_rows_per_src=[10908, 10936, 10778, 10894, 10928, 10998, 10978, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87520 payload_rows_remote=76610 dup_rows=43552 dup_rows_remote=38014 dedup_ratio=0.332275 router_tokens_per_expert=[504, 545, 512, 533, 507, 487, 485, 496, 534, 501, 519, 484, 523, 494, 520, 524, 492, 515, 518, 482, 540, 481, 527, 511, 504, 528, 519, 539, 480, 509, 508, 487, 508, 528, 511, 449, 499, 522, 478, 520, 536, 555, 529, 523, 492, 482, 541, 535, 484, 540, 523, 521, 530, 535, 492, 509, 483, 526, 539, 511, 480, 507, 496, 527, 532, 535, 485, 476, 504, 486, 497, 517, 501, 492, 534, 578, 527, 494, 533, 533, 503, 487, 497, 475, 506, 516, 476, 528, 491, 510, 494, 478, 529, 530, 486, 543, 538, 541, 527, 534, 508, 527, 525, 492, 494, 521, 518, 494, 484, 519, 497, 479, 488, 513, 529, 578, 484, 477, 542, 526, 504, 558, 513, 523, 521, 546, 511, 533] payload_rows_per_dst=[10944, 10908, 10958, 10940, 10910, 10900, 10928, 11032] dup_rows_per_dst=[5392, 5372, 5458, 5466, 5538, 5198, 5468, 5660] combine_rows_per_src=[10944, 10908, 10958, 10940, 10910, 10900, 10928, 11032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=624803840 dispatch_meta_bytes=458784 combine_payload_bytes=624803840 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625262624 ingress_bytes=851296256 link_bytes=851296256 token_bytes=1250066464 weight_movement_bytes=226492416 total_nvlink_bytes=1476558880 weight_to_token_ratio=0.181184 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079563482e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628031488 dispatch_meta_bytes=458192 combine_payload_bytes=628031488 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628489680 ingress_bytes=779026432 link_bytes=779026432 token_bytes=1256521168 weight_movement_bytes=150994944 total_nvlink_bytes=1407516112 weight_to_token_ratio=0.120169 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.879151720e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=458888 combine_payload_bytes=627326976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627785864 ingress_bytes=702824448 link_bytes=702824448 token_bytes=1255112840 weight_movement_bytes=75497472 total_nvlink_bytes=1330610312 weight_to_token_ratio=0.060152 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912803301e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627441664 dispatch_meta_bytes=458784 combine_payload_bytes=627441664 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627900448 ingress_bytes=702939136 link_bytes=702939136 token_bytes=1255342112 weight_movement_bytes=75497472 total_nvlink_bytes=1330839584 weight_to_token_ratio=0.060141 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914257706e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626524160 dispatch_meta_bytes=457832 combine_payload_bytes=626524160 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=626981992 ingress_bytes=777519104 link_bytes=777519104 token_bytes=1253506152 weight_movement_bytes=150994944 total_nvlink_bytes=1404501096 weight_to_token_ratio=0.120458 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.860036680e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625213440 dispatch_meta_bytes=458472 combine_payload_bytes=625213440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625671912 ingress_bytes=700710912 link_bytes=700710912 token_bytes=1250885352 weight_movement_bytes=75497472 total_nvlink_bytes=1326382824 weight_to_token_ratio=0.060355 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.886000689e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87468 payload_rows_remote=76472 dup_rows=43604 dup_rows_remote=37964 dedup_ratio=0.332672 router_tokens_per_expert=[515, 522, 527, 507, 462, 541, 575, 533, 504, 523, 498, 513, 516, 511, 499, 566, 541, 515, 525, 527, 517, 499, 514, 532, 489, 491, 488, 509, 502, 521, 551, 490, 507, 499, 518, 474, 562, 535, 494, 484, 523, 496, 544, 511, 493, 471, 481, 489, 516, 575, 497, 515, 472, 512, 482, 532, 507, 500, 523, 510, 543, 488, 511, 524, 532, 508, 517, 551, 472, 541, 525, 539, 560, 520, 527, 541, 531, 528, 513, 522, 540, 487, 475, 508, 511, 484, 511, 467, 524, 502, 508, 535, 482, 466, 467, 526, 504, 529, 491, 496, 500, 470, 483, 524, 486, 481, 513, 477, 517, 557, 509, 527, 536, 552, 502, 502, 532, 525, 515, 531, 504, 490, 525, 498, 501, 468, 530, 530] payload_rows_per_dst=[10996, 10954, 11086, 11024, 10856, 11024, 10916, 10612] dup_rows_per_dst=[5640, 5468, 5668, 5378, 5406, 5648, 5212, 5184] combine_rows_per_src=[10996, 10954, 11086, 11024, 10856, 11024, 10916, 10612] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627589120 dispatch_meta_bytes=458496 combine_payload_bytes=627589120 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628047616 ingress_bytes=778584064 link_bytes=778584064 token_bytes=1255636736 weight_movement_bytes=150994944 total_nvlink_bytes=1406631680 weight_to_token_ratio=0.120254 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.873541871e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87460 payload_rows_remote=76534 dup_rows=43612 dup_rows_remote=38140 dedup_ratio=0.332733 router_tokens_per_expert=[484, 499, 478, 516, 514, 491, 545, 503, 539, 516, 453, 509, 503, 472, 530, 531, 531, 485, 524, 516, 500, 484, 511, 513, 483, 492, 506, 508, 565, 540, 534, 507, 513, 520, 509, 528, 521, 518, 493, 512, 530, 525, 515, 489, 530, 515, 512, 497, 505, 499, 545, 509, 506, 494, 511, 544, 504, 487, 488, 514, 479, 537, 510, 517, 513, 540, 503, 536, 491, 487, 498, 497, 492, 544, 527, 502, 499, 520, 535, 487, 523, 494, 524, 483, 517, 519, 517, 502, 527, 502, 510, 518, 508, 511, 498, 496, 552, 518, 523, 494, 540, 520, 520, 528, 517, 529, 516, 496, 512, 480, 528, 530, 519, 512, 513, 522, 515, 513, 534, 523, 501, 527, 510, 509, 501, 539, 497, 520] payload_rows_per_dst=[10912, 10926, 10958, 11004, 10908, 10894, 10990, 10868] dup_rows_per_dst=[5254, 5472, 5496, 5294, 5434, 5404, 5616, 5642] combine_rows_per_src=[10912, 10926, 10958, 11004, 10908, 10894, 10990, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87612 payload_rows_remote=76502 dup_rows=43460 dup_rows_remote=37678 dedup_ratio=0.331573 router_tokens_per_expert=[460, 484, 534, 465, 515, 501, 530, 510, 522, 529, 539, 479, 485, 508, 543, 474, 490, 498, 539, 479, 508, 538, 494, 529, 533, 531, 522, 514, 528, 507, 549, 531, 514, 512, 561, 500, 482, 506, 518, 542, 523, 514, 522, 477, 492, 540, 515, 510, 535, 562, 511, 580, 512, 498, 508, 513, 515, 509, 508, 494, 512, 498, 538, 508, 499, 482, 488, 556, 501, 485, 508, 524, 503, 509, 520, 509, 532, 507, 503, 497, 531, 518, 476, 490, 478, 521, 528, 505, 491, 482, 479, 508, 499, 508, 509, 505, 508, 512, 540, 521, 525, 480, 511, 477, 493, 536, 514, 530, 527, 549, 541, 460, 545, 519, 508, 517, 507, 512, 509, 546, 525, 524, 555, 503, 508, 522, 467, 497] payload_rows_per_dst=[10742, 11048, 11058, 10992, 10694, 10806, 11162, 11110] dup_rows_per_dst=[5414, 5532, 5398, 5270, 5186, 5252, 5626, 5782] combine_rows_per_src=[10742, 11048, 11058, 10992, 10694, 10806, 11162, 11110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87658 payload_rows_remote=76884 dup_rows=43414 dup_rows_remote=38166 dedup_ratio=0.331223 router_tokens_per_expert=[490, 510, 540, 510, 519, 521, 517, 517, 435, 526, 498, 524, 509, 475, 544, 518, 482, 497, 486, 486, 469, 549, 557, 534, 503, 506, 490, 493, 516, 496, 525, 488, 505, 485, 542, 495, 546, 530, 552, 522, 528, 527, 519, 509, 481, 539, 517, 460, 515, 527, 498, 516, 523, 497, 508, 521, 545, 533, 544, 499, 529, 518, 518, 477, 573, 509, 589, 520, 507, 513, 482, 529, 551, 490, 525, 528, 508, 496, 520, 522, 508, 491, 481, 497, 529, 474, 504, 545, 519, 571, 514, 537, 478, 533, 530, 491, 481, 504, 508, 470, 493, 479, 517, 504, 476, 496, 491, 495, 520, 514, 525, 538, 553, 517, 517, 517, 524, 518, 543, 498, 544, 463, 519, 478, 494, 500, 487, 534] payload_rows_per_dst=[10842, 10852, 11096, 11024, 11192, 10890, 10774, 10988] dup_rows_per_dst=[5464, 5302, 5418, 5512, 5532, 5514, 5248, 5424] combine_rows_per_src=[10842, 10852, 11096, 11024, 11192, 10890, 10774, 10988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626458624 dispatch_meta_bytes=457744 combine_payload_bytes=626458624 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626916368 ingress_bytes=777453568 link_bytes=777453568 token_bytes=1253374992 weight_movement_bytes=150994944 total_nvlink_bytes=1404369936 weight_to_token_ratio=0.120471 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.859205591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76482 dup_rows=43590 dup_rows_remote=38026 dedup_ratio=0.332565 router_tokens_per_expert=[500, 474, 534, 498, 520, 517, 498, 532, 489, 545, 514, 536, 499, 479, 512, 505, 494, 522, 521, 474, 509, 500, 506, 479, 544, 518, 523, 491, 498, 529, 525, 497, 532, 521, 502, 522, 532, 533, 537, 508, 476, 497, 517, 529, 512, 517, 536, 511, 496, 531, 556, 513, 484, 519, 509, 501, 499, 509, 486, 515, 505, 541, 511, 470, 493, 523, 486, 511, 491, 501, 533, 523, 511, 537, 517, 506, 527, 515, 522, 502, 533, 541, 556, 570, 528, 515, 545, 507, 518, 528, 491, 503, 539, 476, 524, 495, 510, 453, 507, 509, 503, 515, 490, 543, 514, 475, 516, 543, 469, 508, 521, 534, 487, 563, 499, 480, 530, 498, 520, 518, 518, 499, 497, 498, 530, 472, 507, 534] payload_rows_per_dst=[10930, 10868, 11000, 10908, 10924, 11118, 10818, 10916] dup_rows_per_dst=[5374, 5392, 5564, 5382, 5472, 5620, 5402, 5384] combine_rows_per_src=[10930, 10868, 11000, 10908, 10924, 11118, 10818, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87104 payload_rows_remote=76164 dup_rows=43968 dup_rows_remote=38444 dedup_ratio=0.335449 router_tokens_per_expert=[480, 533, 545, 508, 510, 526, 492, 518, 523, 517, 529, 523, 546, 483, 539, 482, 516, 480, 519, 485, 524, 548, 500, 482, 536, 493, 526, 487, 511, 442, 544, 507, 493, 543, 533, 568, 541, 510, 494, 484, 538, 463, 515, 527, 476, 502, 510, 507, 530, 494, 487, 502, 546, 498, 543, 504, 531, 500, 472, 508, 490, 549, 541, 537, 511, 530, 509, 488, 520, 508, 521, 490, 522, 516, 513, 567, 497, 523, 560, 521, 495, 524, 557, 500, 534, 519, 493, 486, 512, 516, 556, 529, 526, 529, 513, 530, 476, 555, 489, 516, 538, 471, 485, 513, 517, 504, 495, 512, 492, 510, 495, 472, 500, 528, 520, 507, 501, 514, 471, 516, 505, 474, 510, 511, 499, 513, 522, 500] payload_rows_per_dst=[10868, 10814, 10940, 10940, 10952, 11046, 10760, 10784] dup_rows_per_dst=[5640, 5386, 5468, 5524, 5640, 5592, 5320, 5398] combine_rows_per_src=[10868, 10814, 10940, 10940, 10952, 11046, 10760, 10784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87500 payload_rows_remote=76604 dup_rows=43572 dup_rows_remote=38012 dedup_ratio=0.332428 router_tokens_per_expert=[521, 526, 517, 543, 499, 513, 521, 509, 488, 498, 517, 518, 513, 525, 506, 510, 520, 514, 515, 511, 523, 520, 526, 506, 525, 516, 509, 523, 495, 501, 568, 503, 542, 477, 557, 537, 484, 516, 495, 489, 480, 492, 518, 473, 495, 507, 513, 504, 562, 566, 512, 514, 550, 487, 529, 495, 510, 482, 538, 473, 490, 513, 501, 535, 543, 505, 500, 523, 555, 493, 542, 504, 507, 522, 500, 539, 529, 478, 512, 493, 497, 508, 519, 503, 534, 509, 527, 506, 486, 489, 494, 529, 542, 547, 522, 516, 468, 520, 502, 523, 507, 532, 560, 492, 541, 534, 493, 506, 471, 474, 525, 542, 543, 504, 462, 494, 451, 540, 507, 449, 532, 470, 508, 532, 532, 497, 519, 498] payload_rows_per_dst=[10936, 10966, 10880, 11014, 10902, 10896, 10970, 10936] dup_rows_per_dst=[5512, 5584, 5278, 5500, 5588, 5560, 5410, 5140] combine_rows_per_src=[10936, 10966, 10880, 11014, 10902, 10896, 10970, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626966528 dispatch_meta_bytes=458696 combine_payload_bytes=626966528 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627425224 ingress_bytes=702464000 link_bytes=702464000 token_bytes=1254391752 weight_movement_bytes=75497472 total_nvlink_bytes=1329889224 weight_to_token_ratio=0.060187 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908232313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626704384 dispatch_meta_bytes=456720 combine_payload_bytes=626704384 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627161104 ingress_bytes=702201856 link_bytes=702201856 token_bytes=1253865488 weight_movement_bytes=75497472 total_nvlink_bytes=1329362960 weight_to_token_ratio=0.060212 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904907958e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87682 payload_rows_remote=76764 dup_rows=43390 dup_rows_remote=38050 dedup_ratio=0.331039 router_tokens_per_expert=[541, 502, 517, 507, 484, 523, 514, 505, 545, 528, 535, 495, 583, 486, 516, 456, 482, 543, 503, 472, 503, 534, 567, 522, 507, 524, 498, 520, 470, 506, 508, 535, 522, 496, 531, 500, 489, 508, 509, 519, 516, 516, 489, 509, 512, 528, 505, 531, 520, 513, 513, 532, 518, 507, 513, 535, 494, 489, 487, 504, 524, 508, 492, 528, 484, 481, 510, 486, 514, 491, 546, 495, 535, 530, 513, 472, 562, 518, 534, 458, 505, 516, 503, 503, 494, 529, 499, 482, 514, 533, 528, 494, 513, 497, 540, 452, 487, 528, 533, 556, 540, 530, 519, 533, 495, 509, 500, 501, 536, 509, 512, 487, 507, 500, 487, 521, 505, 567, 515, 522, 529, 525, 508, 517, 489, 529, 517, 504] payload_rows_per_dst=[10990, 10930, 10978, 10958, 10918, 10916, 11042, 10950] dup_rows_per_dst=[5484, 5458, 5382, 5396, 5340, 5288, 5508, 5534] combine_rows_per_src=[10990, 10930, 10978, 10958, 10918, 10916, 11042, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=629833728 dispatch_meta_bytes=460200 combine_payload_bytes=629833728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630293928 ingress_bytes=705331200 link_bytes=705331200 token_bytes=1260127656 weight_movement_bytes=75497472 total_nvlink_bytes=1335625128 weight_to_token_ratio=0.059913 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.944592445e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626540544 dispatch_meta_bytes=458032 combine_payload_bytes=626540544 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626998576 ingress_bytes=777535488 link_bytes=777535488 token_bytes=1253539120 weight_movement_bytes=150994944 total_nvlink_bytes=1404534064 weight_to_token_ratio=0.120455 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.860244452e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=623935488 dispatch_meta_bytes=458432 combine_payload_bytes=623935488 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624393920 ingress_bytes=774930432 link_bytes=774930432 token_bytes=1248329408 weight_movement_bytes=150994944 total_nvlink_bytes=1399324352 weight_to_token_ratio=0.120958 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.827208675e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=458464 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627998432 ingress_bytes=854032384 link_bytes=854032384 token_bytes=1255538400 weight_movement_bytes=226492416 total_nvlink_bytes=1482030816 weight_to_token_ratio=0.180395 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083033277e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628850688 dispatch_meta_bytes=459256 combine_payload_bytes=628850688 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=629309944 ingress_bytes=855343104 link_bytes=855343104 token_bytes=1258160632 weight_movement_bytes=226492416 total_nvlink_bytes=1484653048 weight_to_token_ratio=0.180019 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084695455e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87458 payload_rows_remote=76388 dup_rows=43614 dup_rows_remote=37974 dedup_ratio=0.332748 router_tokens_per_expert=[546, 531, 518, 496, 531, 525, 506, 528, 515, 557, 534, 480, 467, 528, 511, 470, 517, 519, 531, 540, 480, 509, 510, 535, 502, 509, 551, 525, 508, 500, 490, 480, 525, 497, 506, 508, 525, 509, 526, 504, 548, 502, 522, 527, 517, 490, 491, 483, 521, 532, 491, 523, 493, 507, 528, 467, 525, 498, 489, 469, 491, 542, 540, 544, 532, 508, 488, 519, 525, 527, 483, 507, 485, 503, 531, 555, 525, 530, 500, 542, 539, 532, 497, 510, 530, 495, 506, 477, 489, 512, 496, 540, 523, 510, 493, 517, 555, 519, 502, 548, 500, 491, 515, 543, 468, 513, 506, 514, 525, 516, 470, 528, 491, 501, 477, 513, 545, 492, 491, 513, 504, 504, 492, 495, 538, 493, 544, 515] payload_rows_per_dst=[11070, 11046, 10772, 10926, 10842, 10934, 11028, 10840] dup_rows_per_dst=[5640, 5496, 5386, 5428, 5548, 5342, 5398, 5376] combine_rows_per_src=[11070, 11046, 10772, 10926, 10842, 10934, 11028, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87222 payload_rows_remote=76346 dup_rows=43850 dup_rows_remote=38474 dedup_ratio=0.334549 router_tokens_per_expert=[489, 526, 510, 515, 514, 543, 511, 500, 531, 485, 497, 533, 478, 504, 504, 512, 536, 528, 476, 530, 455, 496, 498, 503, 516, 521, 566, 526, 470, 498, 503, 504, 462, 541, 524, 504, 503, 513, 517, 511, 502, 499, 507, 512, 507, 513, 473, 492, 518, 529, 505, 524, 534, 548, 561, 511, 512, 514, 522, 512, 542, 505, 514, 485, 514, 550, 468, 521, 517, 448, 502, 518, 496, 500, 539, 485, 541, 528, 520, 477, 533, 474, 523, 506, 513, 521, 550, 486, 502, 518, 522, 532, 532, 486, 545, 467, 499, 493, 473, 491, 527, 502, 551, 512, 512, 503, 492, 522, 478, 515, 560, 551, 544, 492, 487, 515, 487, 505, 516, 529, 534, 508, 554, 514, 529, 550, 549, 514] payload_rows_per_dst=[10896, 10876, 10818, 10974, 10848, 10938, 10932, 10940] dup_rows_per_dst=[5408, 5376, 5342, 5698, 5400, 5482, 5430, 5714] combine_rows_per_src=[10896, 10876, 10818, 10974, 10848, 10938, 10932, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76492 dup_rows=43520 dup_rows_remote=37846 dedup_ratio=0.332031 router_tokens_per_expert=[492, 538, 485, 498, 491, 512, 502, 493, 508, 479, 469, 466, 488, 485, 513, 521, 504, 519, 502, 520, 525, 519, 512, 489, 500, 489, 519, 522, 493, 492, 495, 519, 535, 536, 515, 511, 520, 548, 522, 528, 514, 493, 514, 520, 525, 523, 519, 499, 503, 526, 510, 475, 534, 534, 498, 551, 527, 480, 526, 526, 507, 495, 541, 510, 494, 528, 559, 519, 498, 496, 491, 521, 505, 502, 533, 519, 526, 501, 533, 493, 530, 488, 515, 524, 518, 531, 500, 497, 546, 514, 517, 514, 509, 511, 488, 457, 529, 504, 518, 526, 516, 542, 508, 525, 533, 554, 528, 481, 529, 471, 528, 523, 508, 501, 527, 549, 547, 515, 504, 502, 519, 500, 491, 539, 509, 498, 494, 517] payload_rows_per_dst=[10822, 10956, 11022, 10574, 10910, 11316, 10892, 11060] dup_rows_per_dst=[5058, 5282, 5622, 4860, 5490, 6054, 5480, 5674] combine_rows_per_src=[10822, 10956, 11022, 10574, 10910, 11316, 10892, 11060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87162 payload_rows_remote=76214 dup_rows=43910 dup_rows_remote=38404 dedup_ratio=0.335007 router_tokens_per_expert=[516, 517, 509, 488, 490, 507, 523, 502, 505, 548, 479, 530, 518, 516, 526, 510, 518, 513, 560, 536, 489, 492, 498, 485, 507, 499, 527, 503, 491, 512, 494, 496, 454, 527, 544, 492, 511, 498, 528, 504, 513, 490, 498, 492, 504, 516, 491, 527, 539, 529, 530, 523, 520, 519, 472, 532, 473, 521, 536, 517, 504, 470, 525, 496, 496, 569, 558, 531, 542, 513, 494, 513, 526, 532, 522, 529, 512, 484, 474, 511, 552, 467, 533, 520, 518, 428, 531, 521, 515, 506, 476, 470, 500, 505, 494, 523, 499, 506, 537, 534, 508, 580, 478, 507, 519, 497, 524, 480, 526, 539, 490, 503, 538, 509, 511, 461, 542, 563, 535, 495, 524, 493, 528, 494, 557, 527, 528, 540] payload_rows_per_dst=[10906, 10862, 10740, 10720, 10924, 11034, 10948, 11028] dup_rows_per_dst=[5462, 5378, 5438, 5300, 5688, 5476, 5506, 5662] combine_rows_per_src=[10906, 10862, 10740, 10720, 10924, 11034, 10948, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625770496 dispatch_meta_bytes=457448 combine_payload_bytes=625770496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626227944 ingress_bytes=701267968 link_bytes=701267968 token_bytes=1251998440 weight_movement_bytes=75497472 total_nvlink_bytes=1327495912 weight_to_token_ratio=0.060302 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893064944e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87668 payload_rows_remote=76698 dup_rows=43404 dup_rows_remote=38040 dedup_ratio=0.331146 router_tokens_per_expert=[495, 502, 517, 549, 521, 525, 454, 481, 510, 520, 528, 486, 532, 490, 499, 527, 552, 526, 558, 517, 533, 538, 531, 491, 523, 492, 542, 524, 491, 478, 503, 507, 509, 521, 527, 494, 506, 500, 548, 528, 520, 494, 502, 521, 495, 495, 502, 528, 511, 494, 485, 518, 510, 563, 518, 505, 505, 503, 545, 507, 515, 495, 515, 455, 503, 522, 491, 529, 512, 466, 498, 533, 508, 534, 558, 474, 522, 541, 517, 545, 569, 489, 486, 493, 501, 562, 473, 513, 498, 488, 513, 480, 535, 524, 547, 496, 528, 517, 495, 504, 517, 510, 544, 528, 528, 507, 544, 508, 477, 523, 544, 504, 511, 537, 531, 513, 514, 482, 509, 481, 491, 515, 452, 495, 492, 524, 520, 495] payload_rows_per_dst=[10882, 11068, 10956, 10980, 11024, 10970, 10984, 10804] dup_rows_per_dst=[5390, 5544, 5424, 5308, 5482, 5364, 5572, 5320] combine_rows_per_src=[10882, 11068, 10956, 10980, 11024, 10970, 10984, 10804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76466 dup_rows=43546 dup_rows_remote=38066 dedup_ratio=0.332230 router_tokens_per_expert=[537, 489, 544, 461, 576, 509, 541, 537, 507, 541, 525, 516, 495, 490, 504, 510, 504, 518, 537, 531, 551, 498, 517, 533, 511, 511, 512, 490, 524, 504, 541, 489, 562, 527, 513, 508, 499, 514, 518, 506, 525, 504, 512, 535, 506, 483, 522, 536, 511, 512, 512, 523, 541, 512, 534, 553, 475, 505, 541, 544, 524, 478, 523, 506, 521, 534, 534, 511, 525, 498, 497, 476, 551, 482, 544, 488, 502, 474, 533, 475, 467, 519, 475, 479, 511, 543, 507, 480, 501, 488, 465, 528, 541, 509, 493, 509, 478, 530, 526, 526, 518, 522, 496, 502, 565, 524, 475, 481, 473, 525, 467, 532, 522, 518, 465, 521, 523, 534, 502, 516, 529, 512, 474, 503, 500, 536, 491, 473] payload_rows_per_dst=[11042, 10954, 11060, 10952, 11058, 10754, 10858, 10848] dup_rows_per_dst=[5522, 5588, 5480, 5636, 5232, 5276, 5422, 5390] combine_rows_per_src=[11042, 10954, 11060, 10952, 11058, 10754, 10858, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87602 payload_rows_remote=76552 dup_rows=43470 dup_rows_remote=37926 dedup_ratio=0.331650 router_tokens_per_expert=[497, 534, 522, 503, 504, 534, 504, 547, 535, 518, 500, 506, 537, 520, 514, 492, 503, 526, 496, 481, 492, 540, 513, 502, 533, 514, 516, 476, 506, 526, 532, 482, 510, 495, 532, 526, 554, 515, 482, 540, 495, 483, 487, 503, 536, 496, 510, 535, 520, 518, 493, 541, 499, 504, 500, 555, 532, 524, 548, 516, 487, 518, 517, 525, 515, 495, 517, 504, 468, 546, 543, 507, 470, 521, 523, 579, 471, 505, 527, 516, 493, 506, 489, 510, 500, 502, 484, 478, 501, 519, 506, 528, 492, 488, 490, 493, 501, 550, 546, 510, 529, 482, 523, 486, 523, 533, 479, 504, 476, 466, 512, 522, 531, 545, 529, 546, 499, 501, 492, 486, 504, 496, 542, 518, 509, 535, 533, 541] payload_rows_per_dst=[10994, 10866, 10844, 11050, 10976, 10802, 11006, 11064] dup_rows_per_dst=[5540, 5410, 5554, 5544, 5438, 5156, 5278, 5550] combine_rows_per_src=[10994, 10866, 10844, 11050, 10976, 10802, 11006, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625426432 dispatch_meta_bytes=459280 combine_payload_bytes=625426432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625885712 ingress_bytes=700923904 link_bytes=700923904 token_bytes=1251312144 weight_movement_bytes=75497472 total_nvlink_bytes=1326809616 weight_to_token_ratio=0.060335 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888701728e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626622464 dispatch_meta_bytes=457352 combine_payload_bytes=626622464 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627079816 ingress_bytes=777617408 link_bytes=777617408 token_bytes=1253702280 weight_movement_bytes=150994944 total_nvlink_bytes=1404697224 weight_to_token_ratio=0.120439 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861283313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=624345088 dispatch_meta_bytes=458472 combine_payload_bytes=624345088 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624803560 ingress_bytes=775340032 link_bytes=775340032 token_bytes=1249148648 weight_movement_bytes=150994944 total_nvlink_bytes=1400143592 weight_to_token_ratio=0.120878 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.832402979e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628310016 dispatch_meta_bytes=458952 combine_payload_bytes=628310016 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628768968 ingress_bytes=703807488 link_bytes=703807488 token_bytes=1257078984 weight_movement_bytes=75497472 total_nvlink_bytes=1332576456 weight_to_token_ratio=0.060058 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925269632e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76556 dup_rows=43558 dup_rows_remote=38094 dedup_ratio=0.332321 router_tokens_per_expert=[482, 519, 487, 530, 498, 477, 522, 535, 524, 521, 522, 529, 537, 458, 509, 485, 521, 510, 473, 516, 524, 513, 517, 505, 531, 502, 490, 520, 527, 531, 518, 475, 525, 522, 532, 528, 499, 527, 443, 482, 548, 516, 535, 550, 525, 519, 491, 481, 451, 494, 502, 521, 513, 539, 493, 492, 513, 509, 539, 519, 504, 533, 525, 498, 556, 522, 510, 494, 547, 515, 507, 532, 484, 488, 475, 512, 532, 500, 543, 494, 470, 506, 493, 475, 521, 526, 506, 463, 490, 474, 540, 474, 538, 524, 536, 496, 495, 522, 532, 465, 463, 496, 542, 495, 534, 508, 531, 544, 527, 489, 514, 535, 563, 512, 566, 497, 519, 519, 521, 499, 486, 551, 512, 537, 548, 572, 537, 486] payload_rows_per_dst=[10840, 10964, 10960, 10916, 10958, 10816, 10974, 11086] dup_rows_per_dst=[5430, 5382, 5486, 5374, 5464, 5248, 5410, 5764] combine_rows_per_src=[10840, 10964, 10960, 10916, 10958, 10816, 10974, 11086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626409472 dispatch_meta_bytes=458128 combine_payload_bytes=626409472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626867600 ingress_bytes=701906944 link_bytes=701906944 token_bytes=1253277072 weight_movement_bytes=75497472 total_nvlink_bytes=1328774544 weight_to_token_ratio=0.060240 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901168059e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627113984 dispatch_meta_bytes=457912 combine_payload_bytes=627113984 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627571896 ingress_bytes=778108928 link_bytes=778108928 token_bytes=1254685880 weight_movement_bytes=150994944 total_nvlink_bytes=1405680824 weight_to_token_ratio=0.120345 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.867516478e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627146752 dispatch_meta_bytes=458600 combine_payload_bytes=627146752 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=627605352 ingress_bytes=929136640 link_bytes=929136640 token_bytes=1254752104 weight_movement_bytes=301989888 total_nvlink_bytes=1556741992 weight_to_token_ratio=0.240677 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178276045e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87336 payload_rows_remote=76232 dup_rows=43736 dup_rows_remote=37960 dedup_ratio=0.333679 router_tokens_per_expert=[497, 468, 534, 534, 558, 511, 522, 502, 510, 531, 494, 520, 465, 510, 518, 551, 514, 472, 488, 510, 535, 487, 532, 529, 509, 539, 528, 554, 499, 525, 525, 514, 524, 516, 517, 521, 499, 478, 504, 494, 492, 516, 537, 536, 507, 519, 499, 500, 484, 533, 533, 471, 519, 511, 501, 495, 562, 493, 501, 535, 500, 497, 515, 511, 487, 561, 516, 509, 524, 507, 510, 514, 476, 524, 552, 557, 522, 509, 515, 549, 506, 499, 488, 503, 510, 504, 501, 504, 544, 522, 506, 482, 521, 528, 530, 545, 498, 512, 525, 548, 537, 539, 507, 509, 463, 478, 540, 463, 539, 517, 497, 518, 488, 474, 466, 551, 529, 532, 513, 493, 520, 486, 475, 511, 504, 473, 507, 494] payload_rows_per_dst=[11104, 11042, 10896, 11220, 10948, 10892, 10894, 10340] dup_rows_per_dst=[5776, 5524, 5430, 5966, 5232, 5494, 5486, 4828] combine_rows_per_src=[11104, 11042, 10896, 11220, 10948, 10892, 10894, 10340] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87458 payload_rows_remote=76518 dup_rows=43614 dup_rows_remote=38216 dedup_ratio=0.332748 router_tokens_per_expert=[495, 511, 481, 487, 482, 510, 513, 526, 503, 511, 560, 477, 507, 520, 544, 466, 520, 496, 517, 481, 517, 506, 481, 479, 494, 530, 541, 523, 516, 542, 499, 527, 528, 529, 480, 527, 541, 539, 524, 489, 535, 509, 531, 503, 495, 538, 519, 547, 548, 481, 508, 536, 531, 491, 481, 518, 494, 537, 514, 526, 513, 472, 541, 470, 481, 502, 520, 500, 513, 506, 477, 520, 529, 517, 523, 522, 529, 524, 525, 486, 556, 509, 523, 515, 519, 517, 489, 497, 546, 521, 517, 517, 498, 503, 539, 495, 501, 488, 503, 494, 534, 514, 530, 520, 511, 504, 494, 549, 534, 541, 486, 502, 508, 511, 525, 474, 561, 515, 506, 506, 524, 509, 533, 481, 499, 512, 478, 497] payload_rows_per_dst=[10874, 10940, 11002, 10868, 10972, 11040, 10894, 10868] dup_rows_per_dst=[5312, 5398, 5666, 5454, 5376, 5482, 5516, 5410] combine_rows_per_src=[10874, 10940, 11002, 10868, 10972, 11040, 10894, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87504 payload_rows_remote=76500 dup_rows=43568 dup_rows_remote=38158 dedup_ratio=0.332397 router_tokens_per_expert=[513, 527, 509, 511, 509, 549, 522, 508, 520, 527, 521, 488, 542, 529, 510, 487, 513, 548, 497, 542, 439, 481, 467, 462, 498, 481, 501, 509, 525, 520, 486, 532, 491, 494, 499, 496, 528, 507, 524, 528, 496, 515, 507, 489, 554, 514, 519, 507, 479, 524, 523, 498, 500, 535, 487, 500, 529, 528, 550, 523, 499, 527, 508, 540, 510, 507, 509, 513, 531, 544, 526, 499, 533, 516, 532, 516, 519, 513, 520, 491, 486, 533, 530, 488, 530, 544, 487, 508, 537, 490, 550, 489, 525, 534, 488, 512, 507, 540, 506, 530, 518, 521, 531, 539, 503, 519, 481, 520, 492, 478, 518, 504, 517, 537, 503, 484, 455, 514, 488, 514, 513, 481, 544, 463, 549, 542, 490, 534] payload_rows_per_dst=[11012, 10736, 10982, 10900, 11070, 10866, 11004, 10934] dup_rows_per_dst=[5532, 5266, 5354, 5600, 5488, 5596, 5410, 5322] combine_rows_per_src=[11012, 10736, 10982, 10900, 11070, 10866, 11004, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87450 payload_rows_remote=76268 dup_rows=43622 dup_rows_remote=37786 dedup_ratio=0.332809 router_tokens_per_expert=[516, 509, 549, 530, 491, 488, 490, 498, 509, 485, 515, 514, 508, 507, 540, 502, 510, 494, 522, 491, 480, 520, 497, 504, 518, 491, 543, 505, 489, 518, 518, 510, 508, 532, 476, 549, 530, 529, 513, 559, 494, 576, 508, 532, 479, 538, 514, 551, 518, 500, 482, 460, 520, 509, 501, 537, 494, 482, 509, 505, 530, 497, 519, 511, 519, 520, 483, 498, 504, 487, 491, 471, 513, 503, 519, 558, 528, 481, 507, 534, 585, 486, 501, 501, 514, 478, 516, 502, 509, 549, 536, 472, 539, 509, 555, 482, 553, 494, 541, 476, 526, 502, 456, 484, 506, 508, 537, 511, 509, 468, 552, 550, 524, 514, 515, 540, 505, 532, 537, 495, 485, 490, 473, 517, 549, 545, 534, 535] payload_rows_per_dst=[10782, 10910, 10888, 10826, 10932, 10796, 11134, 11182] dup_rows_per_dst=[5208, 5310, 5428, 5322, 5612, 5302, 5604, 5836] combine_rows_per_src=[10782, 10910, 10888, 10826, 10932, 10796, 11134, 11182] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87218 payload_rows_remote=76242 dup_rows=43854 dup_rows_remote=38192 dedup_ratio=0.334579 router_tokens_per_expert=[527, 517, 507, 500, 550, 522, 520, 544, 518, 530, 505, 532, 472, 505, 495, 512, 471, 501, 472, 514, 532, 489, 503, 510, 501, 494, 487, 494, 498, 480, 487, 522, 503, 540, 521, 469, 547, 484, 539, 532, 533, 504, 521, 505, 525, 503, 509, 549, 499, 514, 514, 522, 501, 504, 526, 497, 537, 535, 537, 500, 488, 511, 482, 503, 501, 509, 525, 525, 522, 506, 498, 489, 538, 485, 523, 499, 489, 514, 478, 501, 522, 509, 533, 516, 516, 533, 513, 508, 491, 500, 484, 550, 534, 556, 522, 532, 500, 536, 483, 479, 529, 515, 507, 533, 483, 505, 550, 542, 519, 514, 495, 491, 541, 517, 504, 511, 496, 491, 532, 499, 517, 512, 524, 518, 505, 538, 520, 544] payload_rows_per_dst=[10970, 10758, 10952, 10824, 10856, 10976, 10896, 10986] dup_rows_per_dst=[5542, 5152, 5616, 5516, 5348, 5662, 5466, 5552] combine_rows_per_src=[10970, 10758, 10952, 10824, 10856, 10976, 10896, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=624492544 dispatch_meta_bytes=456768 combine_payload_bytes=624492544 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624949312 ingress_bytes=699990016 link_bytes=699990016 token_bytes=1249441856 weight_movement_bytes=75497472 total_nvlink_bytes=1324939328 weight_to_token_ratio=0.060425 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.876858713e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87246 payload_rows_remote=76370 dup_rows=43826 dup_rows_remote=38378 dedup_ratio=0.334366 router_tokens_per_expert=[492, 519, 535, 518, 502, 519, 520, 511, 499, 515, 507, 504, 472, 489, 489, 557, 498, 523, 520, 523, 511, 512, 511, 521, 554, 513, 527, 483, 515, 537, 498, 498, 510, 502, 514, 527, 491, 543, 562, 502, 482, 465, 516, 519, 524, 525, 514, 466, 507, 465, 512, 516, 506, 508, 535, 520, 509, 517, 553, 517, 516, 500, 516, 499, 498, 532, 519, 527, 511, 499, 529, 497, 508, 488, 496, 516, 527, 531, 464, 519, 481, 510, 557, 522, 514, 510, 524, 550, 503, 533, 557, 496, 505, 470, 528, 517, 483, 485, 513, 529, 471, 501, 514, 492, 471, 523, 520, 537, 530, 545, 526, 504, 532, 515, 506, 501, 473, 516, 508, 493, 522, 502, 519, 508, 507, 531, 535, 536] payload_rows_per_dst=[10848, 10984, 10876, 10872, 10914, 10984, 10928, 10840] dup_rows_per_dst=[5448, 5504, 5448, 5520, 5408, 5570, 5360, 5568] combine_rows_per_src=[10848, 10984, 10876, 10872, 10914, 10984, 10928, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87330 payload_rows_remote=76310 dup_rows=43742 dup_rows_remote=38340 dedup_ratio=0.333725 router_tokens_per_expert=[554, 504, 503, 537, 485, 494, 520, 501, 519, 489, 548, 494, 512, 537, 497, 528, 504, 525, 537, 491, 535, 525, 515, 537, 530, 495, 500, 517, 518, 515, 507, 507, 470, 500, 507, 525, 558, 500, 516, 537, 500, 528, 498, 504, 529, 503, 516, 530, 493, 492, 513, 520, 541, 522, 501, 522, 483, 539, 507, 498, 513, 531, 530, 506, 483, 560, 491, 492, 512, 485, 514, 467, 464, 543, 503, 534, 503, 503, 527, 502, 488, 528, 477, 547, 460, 477, 488, 481, 480, 503, 494, 521, 505, 504, 532, 524, 512, 526, 490, 503, 512, 515, 495, 515, 501, 492, 463, 520, 523, 534, 519, 512, 535, 554, 539, 502, 524, 537, 534, 543, 543, 512, 541, 532, 516, 499, 481, 508] payload_rows_per_dst=[10890, 11038, 11004, 11020, 10792, 10672, 10828, 11086] dup_rows_per_dst=[5554, 5478, 5438, 5402, 5374, 5346, 5436, 5714] combine_rows_per_src=[10890, 11038, 11004, 11020, 10792, 10672, 10828, 11086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626835456 dispatch_meta_bytes=458936 combine_payload_bytes=626835456 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627294392 ingress_bytes=702332928 link_bytes=702332928 token_bytes=1254129848 weight_movement_bytes=75497472 total_nvlink_bytes=1329627320 weight_to_token_ratio=0.060199 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906570135e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626688000 dispatch_meta_bytes=458632 combine_payload_bytes=626688000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627146632 ingress_bytes=702185472 link_bytes=702185472 token_bytes=1253834632 weight_movement_bytes=75497472 total_nvlink_bytes=1329332104 weight_to_token_ratio=0.060213 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904700186e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87494 payload_rows_remote=76562 dup_rows=43578 dup_rows_remote=38012 dedup_ratio=0.332474 router_tokens_per_expert=[530, 518, 501, 528, 570, 519, 505, 494, 532, 490, 493, 512, 541, 508, 521, 451, 508, 497, 501, 536, 504, 482, 526, 507, 535, 507, 543, 526, 507, 498, 523, 498, 527, 502, 490, 500, 544, 485, 465, 511, 476, 474, 522, 511, 533, 495, 545, 483, 514, 541, 482, 488, 499, 546, 565, 524, 477, 522, 506, 533, 550, 500, 537, 531, 524, 522, 511, 527, 526, 531, 525, 507, 525, 529, 517, 508, 485, 515, 549, 448, 500, 492, 473, 473, 509, 539, 489, 505, 506, 512, 509, 534, 543, 510, 546, 541, 535, 510, 484, 497, 508, 477, 509, 520, 483, 503, 490, 507, 506, 487, 579, 536, 517, 492, 492, 490, 540, 536, 505, 549, 502, 482, 549, 469, 510, 509, 515, 529] payload_rows_per_dst=[11004, 10904, 10766, 11112, 10932, 11026, 10864, 10886] dup_rows_per_dst=[5422, 5492, 5360, 5518, 5566, 5336, 5398, 5486] combine_rows_per_src=[11004, 10904, 10766, 11112, 10932, 11026, 10864, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624787456 dispatch_meta_bytes=456216 combine_payload_bytes=624787456 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625243672 ingress_bytes=775782400 link_bytes=775782400 token_bytes=1250031128 weight_movement_bytes=150994944 total_nvlink_bytes=1401026072 weight_to_token_ratio=0.120793 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.838012828e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624574464 dispatch_meta_bytes=457736 combine_payload_bytes=624574464 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625032200 ingress_bytes=775569408 link_bytes=775569408 token_bytes=1249606664 weight_movement_bytes=150994944 total_nvlink_bytes=1400601608 weight_to_token_ratio=0.120834 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.835311790e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625623040 dispatch_meta_bytes=458992 combine_payload_bytes=625623040 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626082032 ingress_bytes=852115456 link_bytes=852115456 token_bytes=1251705072 weight_movement_bytes=226492416 total_nvlink_bytes=1478197488 weight_to_token_ratio=0.180947 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080602343e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625131520 dispatch_meta_bytes=458600 combine_payload_bytes=625131520 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=625590120 ingress_bytes=776126464 link_bytes=776126464 token_bytes=1250721640 weight_movement_bytes=150994944 total_nvlink_bytes=1401716584 weight_to_token_ratio=0.120726 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.842376044e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=458296 combine_payload_bytes=627195904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627654200 ingress_bytes=702693376 link_bytes=702693376 token_bytes=1254850104 weight_movement_bytes=75497472 total_nvlink_bytes=1330347576 weight_to_token_ratio=0.060165 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911141123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87254 payload_rows_remote=76198 dup_rows=43818 dup_rows_remote=38200 dedup_ratio=0.334305 router_tokens_per_expert=[537, 504, 536, 548, 502, 539, 514, 494, 499, 529, 498, 478, 521, 545, 521, 530, 556, 512, 506, 496, 486, 491, 508, 530, 517, 512, 498, 505, 484, 511, 506, 503, 542, 502, 527, 499, 532, 483, 494, 518, 478, 536, 497, 495, 513, 499, 500, 530, 509, 500, 531, 506, 531, 503, 482, 487, 515, 537, 539, 504, 485, 541, 522, 510, 515, 530, 502, 486, 533, 513, 492, 542, 498, 533, 506, 501, 525, 503, 497, 540, 518, 500, 510, 478, 512, 533, 506, 506, 537, 506, 488, 535, 510, 512, 541, 517, 486, 513, 498, 494, 523, 496, 536, 497, 520, 500, 493, 561, 526, 518, 501, 505, 492, 526, 533, 464, 522, 468, 526, 509, 509, 548, 487, 511, 509, 485, 528, 564] payload_rows_per_dst=[11056, 11254, 10882, 10798, 10920, 10822, 10990, 10532] dup_rows_per_dst=[5618, 5948, 5410, 5520, 5512, 5596, 5344, 4870] combine_rows_per_src=[11056, 11254, 10882, 10798, 10920, 10822, 10990, 10532] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87380 payload_rows_remote=76400 dup_rows=43692 dup_rows_remote=38264 dedup_ratio=0.333344 router_tokens_per_expert=[508, 515, 522, 501, 526, 531, 482, 504, 554, 480, 494, 500, 539, 528, 526, 548, 535, 497, 502, 468, 501, 509, 503, 527, 514, 511, 514, 502, 536, 522, 492, 571, 547, 499, 488, 495, 495, 493, 498, 553, 521, 482, 493, 495, 513, 509, 551, 512, 496, 497, 527, 524, 527, 536, 488, 526, 539, 515, 522, 520, 527, 530, 556, 513, 530, 503, 478, 532, 548, 508, 507, 515, 505, 528, 513, 491, 501, 534, 525, 518, 522, 534, 493, 504, 509, 512, 502, 509, 500, 493, 493, 465, 507, 506, 537, 524, 515, 534, 481, 498, 502, 516, 518, 469, 551, 547, 495, 494, 474, 499, 534, 509, 511, 484, 508, 491, 525, 478, 484, 516, 530, 476, 568, 526, 478, 496, 535, 499] payload_rows_per_dst=[10902, 10980, 10936, 10998, 10996, 10906, 10786, 10876] dup_rows_per_dst=[5614, 5428, 5352, 5688, 5476, 5314, 5486, 5334] combine_rows_per_src=[10902, 10980, 10936, 10998, 10996, 10906, 10786, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87274 payload_rows_remote=76226 dup_rows=43798 dup_rows_remote=38098 dedup_ratio=0.334152 router_tokens_per_expert=[506, 512, 534, 471, 497, 535, 513, 474, 482, 490, 531, 505, 513, 489, 546, 490, 511, 552, 494, 532, 523, 547, 481, 507, 510, 528, 508, 498, 503, 526, 599, 495, 522, 503, 502, 507, 503, 546, 503, 510, 516, 548, 487, 503, 487, 500, 529, 505, 508, 496, 478, 528, 519, 500, 512, 490, 486, 530, 514, 479, 481, 508, 520, 505, 480, 534, 500, 508, 478, 510, 534, 495, 516, 491, 509, 510, 549, 497, 512, 503, 510, 509, 504, 515, 531, 535, 514, 514, 479, 529, 529, 517, 517, 508, 498, 505, 516, 558, 546, 470, 520, 516, 532, 532, 561, 535, 450, 563, 523, 516, 512, 524, 508, 527, 496, 491, 511, 520, 507, 528, 529, 491, 524, 514, 486, 530, 525, 508] payload_rows_per_dst=[10780, 11018, 10806, 10854, 10850, 11016, 11048, 10902] dup_rows_per_dst=[5396, 5610, 5536, 5254, 5402, 5412, 5700, 5488] combine_rows_per_src=[10780, 11018, 10806, 10854, 10850, 11016, 11048, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87572 payload_rows_remote=76568 dup_rows=43500 dup_rows_remote=37864 dedup_ratio=0.331879 router_tokens_per_expert=[519, 473, 491, 492, 511, 532, 483, 492, 508, 518, 542, 527, 507, 543, 531, 471, 558, 502, 519, 528, 529, 504, 545, 536, 509, 505, 513, 503, 493, 497, 528, 536, 529, 533, 545, 496, 490, 502, 530, 501, 491, 506, 521, 508, 514, 506, 493, 541, 476, 495, 497, 499, 559, 502, 478, 539, 505, 519, 509, 492, 514, 522, 507, 548, 505, 554, 520, 517, 533, 502, 498, 509, 516, 505, 514, 501, 498, 492, 487, 522, 514, 466, 545, 491, 524, 441, 529, 528, 528, 520, 520, 502, 516, 504, 561, 505, 499, 516, 453, 512, 509, 520, 532, 541, 506, 546, 517, 487, 467, 494, 537, 488, 537, 512, 517, 505, 509, 540, 491, 496, 522, 504, 527, 544, 475, 505, 512, 537] payload_rows_per_dst=[10986, 10942, 10810, 10912, 10964, 11012, 10942, 11004] dup_rows_per_dst=[5294, 5504, 5428, 5410, 5366, 5392, 5470, 5636] combine_rows_per_src=[10986, 10942, 10810, 10912, 10964, 11012, 10942, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624214016 dispatch_meta_bytes=457592 combine_payload_bytes=624214016 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624671608 ingress_bytes=775208960 link_bytes=775208960 token_bytes=1248885624 weight_movement_bytes=150994944 total_nvlink_bytes=1399880568 weight_to_token_ratio=0.120904 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.830740802e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87536 payload_rows_remote=76672 dup_rows=43536 dup_rows_remote=38102 dedup_ratio=0.332153 router_tokens_per_expert=[531, 481, 536, 477, 517, 471, 504, 524, 535, 485, 517, 494, 520, 497, 486, 469, 494, 516, 521, 514, 496, 534, 493, 519, 497, 530, 454, 510, 523, 514, 550, 515, 521, 510, 522, 507, 510, 530, 487, 554, 480, 524, 498, 502, 513, 502, 560, 479, 480, 523, 481, 477, 490, 545, 526, 522, 492, 498, 541, 509, 490, 477, 523, 513, 519, 554, 469, 522, 573, 520, 523, 511, 513, 507, 505, 493, 519, 525, 551, 541, 471, 530, 512, 534, 564, 505, 504, 477, 526, 486, 475, 524, 480, 526, 525, 510, 511, 527, 553, 505, 480, 542, 494, 556, 507, 500, 531, 468, 493, 518, 502, 473, 557, 514, 503, 519, 530, 520, 562, 488, 501, 530, 542, 517, 506, 501, 544, 538] payload_rows_per_dst=[10824, 10974, 10980, 10738, 11052, 10864, 10962, 11142] dup_rows_per_dst=[5264, 5386, 5418, 5436, 5638, 5434, 5358, 5602] combine_rows_per_src=[10824, 10974, 10980, 10738, 11052, 10864, 10962, 11142] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87692 payload_rows_remote=76604 dup_rows=43380 dup_rows_remote=38000 dedup_ratio=0.330963 router_tokens_per_expert=[503, 514, 510, 542, 502, 548, 499, 509, 558, 505, 527, 499, 482, 534, 490, 517, 513, 505, 555, 512, 507, 510, 515, 517, 520, 484, 514, 513, 504, 504, 490, 487, 555, 553, 542, 514, 526, 484, 461, 484, 504, 508, 511, 521, 522, 525, 518, 506, 481, 512, 553, 523, 480, 482, 495, 522, 547, 480, 506, 504, 505, 522, 502, 501, 511, 536, 530, 528, 498, 530, 531, 526, 574, 487, 499, 493, 495, 532, 495, 495, 491, 494, 553, 512, 488, 537, 522, 482, 500, 521, 485, 501, 492, 532, 515, 537, 516, 500, 512, 504, 502, 490, 531, 507, 519, 512, 527, 499, 510, 515, 495, 518, 507, 484, 508, 542, 501, 533, 537, 491, 505, 520, 502, 529, 502, 481, 549, 528] payload_rows_per_dst=[11042, 10860, 11088, 10874, 10940, 10978, 10868, 11042] dup_rows_per_dst=[5436, 5440, 5380, 5356, 5580, 5346, 5446, 5396] combine_rows_per_src=[11042, 10860, 11088, 10874, 10940, 10978, 10868, 11042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76640 dup_rows=43534 dup_rows_remote=38010 dedup_ratio=0.332138 router_tokens_per_expert=[487, 525, 472, 552, 527, 516, 516, 483, 522, 537, 547, 531, 522, 512, 561, 531, 514, 535, 518, 530, 487, 501, 490, 485, 513, 550, 470, 545, 500, 526, 505, 496, 514, 477, 535, 509, 527, 471, 513, 450, 491, 504, 503, 481, 532, 514, 481, 505, 522, 522, 486, 526, 531, 508, 527, 508, 558, 530, 487, 523, 474, 531, 459, 519, 505, 537, 501, 486, 566, 515, 542, 489, 503, 528, 508, 473, 476, 550, 528, 493, 518, 516, 496, 520, 521, 488, 518, 559, 495, 500, 539, 565, 516, 505, 505, 526, 517, 509, 475, 511, 498, 515, 483, 565, 535, 503, 489, 516, 487, 538, 532, 497, 535, 518, 507, 511, 490, 525, 494, 536, 480, 513, 507, 483, 520, 521, 505, 510] payload_rows_per_dst=[11092, 10912, 10816, 10898, 10992, 11014, 10840, 10974] dup_rows_per_dst=[5590, 5418, 5198, 5524, 5408, 5560, 5500, 5336] combine_rows_per_src=[11092, 10912, 10816, 10898, 10992, 11014, 10840, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76454 dup_rows=43738 dup_rows_remote=38184 dedup_ratio=0.333694 router_tokens_per_expert=[484, 525, 528, 440, 504, 500, 490, 539, 507, 561, 490, 521, 542, 503, 526, 511, 548, 471, 500, 512, 531, 534, 510, 521, 524, 501, 470, 503, 485, 509, 518, 521, 501, 535, 545, 522, 523, 541, 549, 527, 538, 513, 517, 528, 490, 532, 545, 498, 520, 556, 524, 490, 511, 486, 514, 466, 495, 525, 493, 494, 517, 505, 532, 539, 506, 517, 516, 528, 484, 547, 533, 512, 527, 508, 486, 508, 484, 530, 516, 515, 533, 506, 560, 504, 495, 478, 516, 545, 513, 532, 511, 501, 512, 512, 482, 499, 498, 471, 517, 481, 526, 543, 519, 529, 517, 510, 493, 503, 497, 531, 541, 522, 514, 518, 487, 516, 529, 512, 471, 504, 527, 441, 522, 492, 514, 498, 494, 483] payload_rows_per_dst=[10816, 10916, 11128, 11018, 10880, 10924, 10900, 10752] dup_rows_per_dst=[5526, 5400, 5680, 5316, 5554, 5474, 5496, 5292] combine_rows_per_src=[10816, 10916, 11128, 11018, 10880, 10924, 10900, 10752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625868800 dispatch_meta_bytes=458656 combine_payload_bytes=625868800 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626327456 ingress_bytes=776863744 link_bytes=776863744 token_bytes=1252196256 weight_movement_bytes=150994944 total_nvlink_bytes=1403191200 weight_to_token_ratio=0.120584 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851725792e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=624443392 dispatch_meta_bytes=457296 combine_payload_bytes=624443392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624900688 ingress_bytes=699940864 link_bytes=699940864 token_bytes=1249344080 weight_movement_bytes=75497472 total_nvlink_bytes=1324841552 weight_to_token_ratio=0.060430 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.876235397e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627245056 dispatch_meta_bytes=457728 combine_payload_bytes=627245056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627702784 ingress_bytes=702742528 link_bytes=702742528 token_bytes=1254947840 weight_movement_bytes=75497472 total_nvlink_bytes=1330445312 weight_to_token_ratio=0.060160 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911764440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628097024 dispatch_meta_bytes=459096 combine_payload_bytes=628097024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628556120 ingress_bytes=703594496 link_bytes=703594496 token_bytes=1256653144 weight_movement_bytes=75497472 total_nvlink_bytes=1332150616 weight_to_token_ratio=0.060078 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922568593e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=458416 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627998384 ingress_bytes=854032384 link_bytes=854032384 token_bytes=1255538352 weight_movement_bytes=226492416 total_nvlink_bytes=1482030768 weight_to_token_ratio=0.180395 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083033277e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627834880 dispatch_meta_bytes=458600 combine_payload_bytes=627834880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628293480 ingress_bytes=703332352 link_bytes=703332352 token_bytes=1256128360 weight_movement_bytes=75497472 total_nvlink_bytes=1331625832 weight_to_token_ratio=0.060103 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919244238e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626311168 dispatch_meta_bytes=458552 combine_payload_bytes=626311168 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=626769720 ingress_bytes=928301056 link_bytes=928301056 token_bytes=1253080888 weight_movement_bytes=301989888 total_nvlink_bytes=1555070776 weight_to_token_ratio=0.240998 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177216407e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87524 payload_rows_remote=76536 dup_rows=43548 dup_rows_remote=38026 dedup_ratio=0.332245 router_tokens_per_expert=[553, 525, 523, 504, 513, 505, 466, 527, 508, 532, 522, 512, 522, 494, 508, 520, 510, 534, 510, 481, 481, 500, 490, 503, 528, 520, 535, 520, 492, 455, 546, 514, 525, 512, 466, 488, 500, 527, 537, 510, 532, 505, 487, 537, 570, 524, 568, 488, 516, 544, 495, 508, 522, 490, 522, 527, 529, 520, 504, 531, 537, 491, 515, 500, 500, 506, 489, 489, 525, 502, 516, 548, 514, 513, 517, 540, 517, 498, 508, 478, 532, 498, 549, 507, 516, 495, 509, 493, 509, 513, 506, 538, 489, 536, 500, 523, 486, 524, 515, 468, 491, 487, 524, 521, 483, 495, 564, 516, 480, 553, 507, 521, 522, 498, 469, 513, 483, 501, 493, 466, 580, 504, 494, 518, 504, 572, 493, 538] payload_rows_per_dst=[10988, 10808, 11056, 11106, 10748, 11000, 10896, 10922] dup_rows_per_dst=[5522, 5476, 5460, 5690, 5226, 5426, 5374, 5374] combine_rows_per_src=[10988, 10808, 11056, 11106, 10748, 11000, 10896, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87442 payload_rows_remote=76526 dup_rows=43630 dup_rows_remote=38434 dedup_ratio=0.332870 router_tokens_per_expert=[500, 515, 551, 479, 518, 546, 486, 480, 520, 526, 505, 488, 506, 523, 544, 509, 498, 524, 510, 503, 500, 531, 508, 490, 528, 494, 494, 507, 509, 464, 496, 500, 524, 502, 461, 537, 530, 515, 550, 494, 486, 505, 485, 518, 464, 497, 481, 512, 537, 531, 514, 532, 521, 552, 527, 495, 514, 477, 458, 527, 475, 482, 571, 485, 508, 505, 519, 506, 519, 507, 521, 541, 513, 478, 479, 529, 579, 535, 556, 540, 513, 518, 530, 490, 504, 527, 497, 528, 525, 480, 480, 562, 564, 534, 508, 441, 493, 493, 489, 528, 516, 499, 527, 530, 540, 536, 499, 494, 546, 524, 504, 545, 532, 482, 529, 529, 511, 493, 495, 530, 539, 496, 538, 529, 508, 481, 516, 518] payload_rows_per_dst=[11008, 10916, 10750, 10884, 11064, 10874, 10976, 10970] dup_rows_per_dst=[5384, 5196, 5372, 5512, 5606, 5528, 5550, 5482] combine_rows_per_src=[11008, 10916, 10750, 10884, 11064, 10874, 10976, 10970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87788 payload_rows_remote=76790 dup_rows=43284 dup_rows_remote=37866 dedup_ratio=0.330231 router_tokens_per_expert=[499, 531, 492, 504, 497, 470, 535, 503, 486, 500, 539, 536, 542, 527, 475, 476, 497, 480, 529, 537, 496, 526, 547, 464, 540, 511, 540, 519, 540, 582, 491, 524, 487, 470, 504, 526, 535, 516, 525, 498, 528, 504, 519, 499, 526, 516, 541, 516, 515, 519, 501, 513, 495, 499, 490, 503, 490, 519, 524, 494, 503, 545, 514, 521, 519, 505, 498, 531, 481, 518, 530, 521, 503, 518, 479, 505, 529, 520, 515, 510, 510, 513, 474, 495, 537, 512, 511, 485, 558, 509, 543, 524, 496, 519, 527, 487, 495, 535, 505, 496, 527, 512, 499, 526, 462, 496, 519, 527, 532, 526, 521, 530, 500, 520, 531, 541, 483, 503, 520, 532, 482, 487, 545, 447, 514, 511, 526, 514] payload_rows_per_dst=[10906, 11018, 11082, 10930, 10932, 10962, 10998, 10960] dup_rows_per_dst=[5318, 5628, 5338, 5360, 5432, 5438, 5418, 5352] combine_rows_per_src=[10906, 11018, 11082, 10930, 10932, 10962, 10998, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76692 dup_rows=43454 dup_rows_remote=37964 dedup_ratio=0.331528 router_tokens_per_expert=[477, 548, 533, 505, 515, 505, 519, 500, 478, 532, 497, 473, 469, 504, 488, 511, 501, 470, 522, 526, 510, 529, 503, 493, 531, 486, 532, 483, 508, 463, 532, 512, 497, 498, 538, 518, 477, 540, 551, 543, 511, 513, 541, 551, 522, 481, 536, 569, 540, 515, 468, 499, 519, 472, 524, 500, 502, 531, 529, 530, 514, 505, 498, 520, 527, 480, 497, 512, 525, 477, 515, 482, 497, 500, 494, 488, 562, 475, 508, 542, 482, 486, 493, 533, 594, 509, 519, 547, 541, 527, 529, 536, 537, 507, 506, 458, 526, 528, 530, 524, 529, 497, 487, 536, 508, 546, 526, 485, 558, 491, 505, 520, 529, 478, 483, 528, 547, 492, 503, 497, 500, 505, 513, 520, 504, 499, 497, 553] payload_rows_per_dst=[10872, 10906, 11034, 10828, 10990, 11018, 11044, 10926] dup_rows_per_dst=[5236, 5296, 5584, 5252, 5424, 5590, 5582, 5490] combine_rows_per_src=[10872, 10906, 11034, 10828, 10990, 11018, 11044, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87808 payload_rows_remote=76780 dup_rows=43264 dup_rows_remote=38076 dedup_ratio=0.330078 router_tokens_per_expert=[455, 506, 458, 529, 512, 498, 555, 508, 491, 480, 526, 515, 502, 544, 514, 487, 497, 527, 521, 496, 496, 489, 514, 519, 506, 508, 539, 542, 524, 526, 479, 512, 528, 523, 573, 502, 533, 510, 529, 533, 491, 547, 488, 487, 532, 543, 487, 534, 511, 527, 473, 519, 550, 483, 522, 509, 551, 472, 497, 532, 517, 524, 513, 515, 539, 499, 527, 512, 497, 536, 529, 532, 499, 520, 510, 492, 540, 519, 507, 486, 521, 461, 489, 512, 523, 494, 509, 532, 520, 520, 517, 479, 492, 530, 485, 524, 561, 490, 483, 493, 512, 502, 494, 501, 551, 516, 526, 493, 478, 514, 523, 496, 556, 492, 484, 524, 461, 529, 527, 497, 513, 540, 545, 509, 531, 516, 495, 502] payload_rows_per_dst=[10930, 11026, 11070, 10968, 10932, 11028, 10828, 11026] dup_rows_per_dst=[5230, 5364, 5610, 5462, 5556, 5188, 5438, 5416] combine_rows_per_src=[10930, 11026, 11070, 10968, 10932, 11028, 10828, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626982912 dispatch_meta_bytes=458248 combine_payload_bytes=626982912 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627441160 ingress_bytes=702480384 link_bytes=702480384 token_bytes=1254424072 weight_movement_bytes=75497472 total_nvlink_bytes=1329921544 weight_to_token_ratio=0.060185 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908440085e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87258 payload_rows_remote=76448 dup_rows=43814 dup_rows_remote=38474 dedup_ratio=0.334274 router_tokens_per_expert=[469, 498, 521, 519, 487, 500, 487, 529, 532, 475, 514, 529, 530, 552, 481, 542, 561, 478, 489, 521, 486, 553, 535, 501, 528, 519, 520, 536, 508, 536, 513, 515, 486, 510, 527, 486, 504, 487, 513, 512, 548, 491, 491, 488, 514, 506, 526, 486, 495, 523, 528, 536, 530, 495, 537, 555, 480, 498, 510, 517, 475, 485, 531, 505, 512, 522, 529, 548, 503, 544, 496, 525, 518, 499, 505, 496, 496, 520, 517, 546, 489, 578, 557, 520, 552, 521, 497, 508, 525, 544, 506, 513, 538, 511, 511, 498, 503, 474, 525, 511, 529, 536, 470, 539, 511, 490, 489, 461, 514, 503, 496, 478, 525, 507, 468, 490, 470, 482, 546, 522, 519, 521, 520, 516, 532, 525, 473, 508] payload_rows_per_dst=[10870, 11032, 10810, 10964, 10896, 10984, 10796, 10906] dup_rows_per_dst=[5460, 5566, 5340, 5436, 5656, 5752, 5262, 5342] combine_rows_per_src=[10870, 11032, 10810, 10964, 10896, 10984, 10796, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87432 payload_rows_remote=76484 dup_rows=43640 dup_rows_remote=38038 dedup_ratio=0.332947 router_tokens_per_expert=[498, 512, 524, 533, 531, 534, 503, 519, 512, 508, 529, 491, 532, 502, 548, 485, 516, 507, 490, 470, 513, 509, 552, 508, 516, 475, 535, 513, 480, 513, 475, 542, 491, 547, 510, 521, 529, 526, 510, 516, 527, 525, 496, 529, 531, 516, 528, 505, 532, 531, 495, 561, 526, 519, 511, 492, 517, 489, 494, 492, 508, 509, 542, 557, 500, 521, 492, 492, 512, 518, 510, 482, 484, 510, 504, 496, 507, 499, 527, 513, 509, 494, 511, 502, 537, 509, 486, 507, 508, 503, 524, 500, 491, 506, 521, 487, 508, 529, 493, 532, 519, 533, 527, 503, 480, 483, 536, 527, 523, 519, 512, 509, 497, 528, 523, 463, 515, 561, 552, 511, 499, 480, 503, 558, 501, 499, 487, 507] payload_rows_per_dst=[11062, 10864, 11036, 10948, 10830, 10864, 10950, 10878] dup_rows_per_dst=[5460, 5364, 5578, 5602, 5304, 5326, 5516, 5490] combine_rows_per_src=[11062, 10864, 11036, 10948, 10830, 10864, 10950, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87276 payload_rows_remote=76388 dup_rows=43796 dup_rows_remote=38238 dedup_ratio=0.334137 router_tokens_per_expert=[482, 530, 512, 524, 503, 461, 493, 522, 502, 545, 524, 487, 490, 514, 521, 486, 486, 487, 506, 464, 486, 494, 472, 504, 502, 506, 539, 550, 537, 490, 559, 521, 523, 516, 522, 478, 532, 514, 489, 513, 478, 515, 532, 536, 513, 474, 500, 528, 492, 510, 493, 558, 490, 527, 491, 490, 531, 518, 509, 542, 489, 508, 536, 550, 581, 514, 489, 489, 520, 522, 516, 473, 517, 502, 533, 470, 502, 539, 514, 542, 533, 487, 526, 512, 484, 484, 536, 488, 568, 526, 515, 552, 536, 506, 537, 497, 490, 516, 526, 514, 528, 519, 532, 533, 455, 525, 505, 485, 481, 526, 535, 507, 542, 510, 536, 487, 518, 503, 531, 487, 540, 519, 507, 493, 502, 521, 541, 516] payload_rows_per_dst=[10870, 10876, 10902, 10892, 10888, 10970, 10838, 11040] dup_rows_per_dst=[5322, 5330, 5424, 5576, 5558, 5604, 5516, 5466] combine_rows_per_src=[10870, 10876, 10902, 10892, 10888, 10970, 10838, 11040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626900992 dispatch_meta_bytes=459840 combine_payload_bytes=626900992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627360832 ingress_bytes=702398464 link_bytes=702398464 token_bytes=1254261824 weight_movement_bytes=75497472 total_nvlink_bytes=1329759296 weight_to_token_ratio=0.060193 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907401224e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=629063680 dispatch_meta_bytes=458624 combine_payload_bytes=629063680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629522304 ingress_bytes=704561152 link_bytes=704561152 token_bytes=1258585984 weight_movement_bytes=75497472 total_nvlink_bytes=1334083456 weight_to_token_ratio=0.059986 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.934827152e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628260864 dispatch_meta_bytes=458624 combine_payload_bytes=628260864 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628719488 ingress_bytes=703758336 link_bytes=703758336 token_bytes=1256980352 weight_movement_bytes=75497472 total_nvlink_bytes=1332477824 weight_to_token_ratio=0.060063 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.049632 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924646315e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628981760 dispatch_meta_bytes=459424 combine_payload_bytes=628981760 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=629441184 ingress_bytes=855474176 link_bytes=855474176 token_bytes=1258422944 weight_movement_bytes=226492416 total_nvlink_bytes=1484915360 weight_to_token_ratio=0.179981 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084861672e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626262016 dispatch_meta_bytes=459688 combine_payload_bytes=626262016 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626721704 ingress_bytes=852754432 link_bytes=852754432 token_bytes=1252983720 weight_movement_bytes=226492416 total_nvlink_bytes=1479476136 weight_to_token_ratio=0.180762 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081412654e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=458088 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627015016 ingress_bytes=777551872 link_bytes=777551872 token_bytes=1253571944 weight_movement_bytes=150994944 total_nvlink_bytes=1404566888 weight_to_token_ratio=0.120452 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.860452224e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625770496 dispatch_meta_bytes=458504 combine_payload_bytes=625770496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626229000 ingress_bytes=776765440 link_bytes=776765440 token_bytes=1251999496 weight_movement_bytes=150994944 total_nvlink_bytes=1402994440 weight_to_token_ratio=0.120603 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850479159e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87476 payload_rows_remote=76208 dup_rows=43596 dup_rows_remote=37720 dedup_ratio=0.332611 router_tokens_per_expert=[466, 494, 543, 553, 504, 550, 452, 546, 510, 516, 506, 524, 522, 513, 527, 522, 508, 507, 499, 543, 518, 536, 503, 538, 515, 502, 557, 507, 504, 474, 490, 508, 493, 473, 463, 510, 527, 519, 526, 508, 480, 499, 480, 549, 552, 525, 495, 466, 504, 500, 487, 502, 529, 512, 519, 533, 512, 522, 490, 503, 499, 541, 534, 500, 520, 517, 518, 494, 505, 525, 524, 546, 571, 529, 540, 514, 519, 518, 479, 507, 476, 483, 544, 546, 519, 519, 472, 509, 507, 518, 520, 528, 490, 511, 478, 537, 516, 510, 447, 497, 507, 529, 536, 503, 500, 512, 498, 486, 548, 562, 537, 492, 540, 530, 531, 501, 502, 518, 525, 496, 496, 483, 526, 505, 474, 536, 492, 509] payload_rows_per_dst=[11268, 11016, 10890, 10876, 10698, 10844, 10946, 10938] dup_rows_per_dst=[5876, 5674, 5252, 5582, 5306, 5102, 5414, 5390] combine_rows_per_src=[11268, 11016, 10890, 10876, 10698, 10844, 10946, 10938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87322 payload_rows_remote=76314 dup_rows=43750 dup_rows_remote=38400 dedup_ratio=0.333786 router_tokens_per_expert=[552, 512, 497, 538, 482, 520, 510, 488, 507, 532, 484, 523, 479, 500, 534, 511, 534, 474, 495, 487, 506, 493, 509, 492, 530, 529, 514, 498, 539, 510, 530, 539, 564, 515, 512, 521, 520, 543, 499, 524, 536, 508, 486, 488, 529, 527, 467, 472, 508, 525, 516, 504, 486, 542, 501, 524, 500, 511, 537, 509, 497, 523, 496, 544, 506, 472, 519, 496, 521, 509, 572, 554, 500, 497, 522, 508, 511, 546, 516, 506, 504, 542, 522, 508, 495, 504, 476, 530, 538, 508, 490, 493, 514, 523, 533, 497, 526, 524, 494, 476, 568, 528, 499, 526, 504, 548, 477, 492, 513, 473, 504, 533, 504, 494, 518, 504, 489, 500, 476, 469, 499, 478, 519, 551, 514, 567, 558, 497] payload_rows_per_dst=[10964, 11008, 10882, 10932, 10924, 10838, 10990, 10784] dup_rows_per_dst=[5374, 5350, 5540, 5514, 5586, 5516, 5380, 5490] combine_rows_per_src=[10964, 11008, 10882, 10932, 10924, 10838, 10990, 10784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87194 payload_rows_remote=76282 dup_rows=43878 dup_rows_remote=38172 dedup_ratio=0.334763 router_tokens_per_expert=[497, 536, 505, 563, 509, 526, 499, 496, 500, 499, 521, 517, 537, 494, 529, 535, 490, 506, 516, 489, 534, 467, 547, 527, 495, 493, 481, 486, 532, 522, 555, 538, 495, 513, 516, 555, 506, 512, 512, 497, 511, 512, 524, 529, 517, 490, 470, 505, 539, 510, 511, 515, 518, 497, 503, 522, 505, 496, 503, 561, 505, 484, 534, 486, 506, 504, 498, 537, 537, 478, 548, 532, 507, 532, 494, 529, 499, 500, 503, 465, 513, 513, 504, 527, 517, 472, 514, 484, 508, 501, 456, 537, 492, 512, 550, 533, 515, 499, 518, 499, 520, 549, 512, 496, 500, 530, 527, 517, 530, 513, 548, 495, 489, 542, 543, 502, 496, 519, 510, 485, 501, 496, 502, 511, 507, 544, 528, 497] payload_rows_per_dst=[10744, 10930, 11108, 10482, 11144, 10904, 10970, 10912] dup_rows_per_dst=[5400, 5426, 5544, 4852, 5964, 5380, 5606, 5706] combine_rows_per_src=[10744, 10930, 11108, 10482, 11144, 10904, 10970, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87726 payload_rows_remote=76746 dup_rows=43346 dup_rows_remote=38058 dedup_ratio=0.330704 router_tokens_per_expert=[533, 555, 512, 540, 521, 505, 499, 481, 471, 524, 543, 557, 509, 477, 513, 510, 504, 534, 494, 513, 506, 521, 538, 514, 516, 494, 509, 497, 539, 506, 504, 519, 530, 505, 503, 511, 534, 497, 534, 513, 524, 514, 546, 542, 499, 535, 496, 558, 505, 533, 530, 529, 489, 474, 570, 503, 502, 510, 543, 476, 514, 542, 511, 507, 511, 506, 515, 525, 488, 474, 526, 500, 471, 487, 510, 504, 523, 511, 516, 530, 507, 521, 500, 528, 552, 535, 500, 502, 522, 529, 515, 512, 523, 466, 475, 498, 537, 473, 520, 467, 517, 502, 499, 556, 507, 498, 544, 479, 516, 512, 485, 522, 490, 508, 470, 522, 568, 469, 520, 519, 507, 473, 514, 477, 537, 518, 477, 514] payload_rows_per_dst=[10946, 10974, 11064, 11056, 10924, 10942, 10980, 10840] dup_rows_per_dst=[5554, 5442, 5618, 5420, 5270, 5428, 5288, 5326] combine_rows_per_src=[10946, 10974, 11064, 11056, 10924, 10942, 10980, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87516 payload_rows_remote=76564 dup_rows=43556 dup_rows_remote=38140 dedup_ratio=0.332306 router_tokens_per_expert=[503, 511, 543, 504, 527, 512, 502, 523, 531, 523, 520, 484, 538, 499, 514, 509, 460, 469, 482, 526, 539, 523, 483, 532, 550, 486, 513, 550, 509, 470, 554, 500, 559, 524, 511, 529, 505, 503, 498, 488, 515, 523, 495, 484, 496, 497, 489, 535, 522, 481, 524, 517, 550, 521, 555, 488, 513, 523, 469, 537, 506, 503, 501, 488, 523, 511, 531, 456, 521, 513, 470, 470, 516, 505, 542, 507, 504, 496, 535, 479, 514, 514, 512, 540, 496, 503, 486, 515, 516, 501, 534, 523, 508, 510, 507, 505, 510, 536, 504, 472, 489, 556, 533, 511, 494, 533, 501, 546, 533, 527, 497, 509, 558, 507, 490, 508, 529, 513, 496, 498, 517, 565, 550, 513, 534, 519, 491, 496] payload_rows_per_dst=[10952, 10858, 10944, 10938, 10892, 10952, 10964, 11016] dup_rows_per_dst=[5534, 5434, 5358, 5458, 5266, 5416, 5538, 5552] combine_rows_per_src=[10952, 10858, 10944, 10938, 10892, 10952, 10964, 11016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=624295936 dispatch_meta_bytes=455712 combine_payload_bytes=624295936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624751648 ingress_bytes=699793408 link_bytes=699793408 token_bytes=1249047584 weight_movement_bytes=75497472 total_nvlink_bytes=1324545056 weight_to_token_ratio=0.060444 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.874365447e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87300 payload_rows_remote=76508 dup_rows=43772 dup_rows_remote=38354 dedup_ratio=0.333954 router_tokens_per_expert=[531, 549, 559, 465, 496, 518, 507, 544, 501, 482, 509, 498, 523, 488, 559, 505, 521, 527, 505, 476, 492, 478, 499, 525, 509, 550, 513, 472, 527, 509, 533, 471, 502, 529, 511, 532, 498, 494, 519, 536, 489, 511, 501, 513, 483, 515, 491, 481, 462, 536, 505, 519, 552, 518, 504, 491, 514, 509, 509, 546, 511, 522, 563, 568, 506, 499, 521, 504, 453, 500, 495, 499, 478, 525, 478, 510, 512, 471, 512, 523, 523, 506, 544, 527, 543, 466, 488, 498, 499, 509, 526, 522, 521, 529, 543, 532, 541, 526, 503, 505, 548, 500, 544, 549, 490, 532, 509, 518, 510, 476, 511, 494, 518, 538, 531, 509, 489, 498, 542, 513, 511, 488, 514, 548, 495, 532, 495, 522] payload_rows_per_dst=[10916, 10918, 10792, 11012, 10776, 11020, 10866, 11000] dup_rows_per_dst=[5552, 5296, 5418, 5646, 5196, 5532, 5646, 5486] combine_rows_per_src=[10916, 10918, 10792, 11012, 10776, 11020, 10866, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87492 payload_rows_remote=76516 dup_rows=43580 dup_rows_remote=37876 dedup_ratio=0.332489 router_tokens_per_expert=[510, 512, 498, 501, 498, 483, 460, 521, 488, 494, 481, 519, 523, 531, 496, 543, 532, 481, 468, 537, 536, 511, 518, 535, 512, 516, 510, 499, 510, 518, 488, 485, 517, 513, 469, 519, 518, 507, 524, 526, 479, 494, 522, 513, 498, 508, 519, 471, 559, 515, 484, 528, 479, 524, 535, 494, 533, 516, 517, 526, 563, 514, 534, 519, 503, 478, 497, 506, 536, 497, 495, 510, 493, 544, 508, 513, 546, 538, 499, 531, 519, 525, 505, 504, 502, 529, 539, 510, 474, 506, 532, 509, 523, 519, 525, 513, 532, 496, 513, 515, 508, 508, 522, 503, 500, 512, 531, 524, 504, 533, 503, 529, 522, 488, 538, 498, 573, 502, 514, 515, 530, 517, 489, 519, 489, 484, 518, 528] payload_rows_per_dst=[10772, 11024, 10820, 10976, 10930, 11014, 10952, 11004] dup_rows_per_dst=[5344, 5288, 5374, 5704, 5458, 5454, 5514, 5444] combine_rows_per_src=[10772, 11024, 10820, 10976, 10930, 11014, 10952, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87624 payload_rows_remote=76634 dup_rows=43448 dup_rows_remote=37852 dedup_ratio=0.331482 router_tokens_per_expert=[493, 503, 494, 498, 533, 508, 543, 507, 506, 540, 500, 481, 501, 532, 508, 556, 518, 495, 508, 538, 530, 538, 491, 496, 502, 516, 472, 542, 519, 529, 520, 489, 515, 482, 515, 469, 504, 487, 502, 541, 499, 497, 497, 469, 516, 523, 505, 545, 505, 513, 498, 550, 512, 480, 511, 546, 521, 494, 519, 529, 506, 507, 488, 526, 525, 530, 511, 502, 558, 493, 521, 534, 513, 522, 545, 508, 536, 539, 487, 469, 509, 492, 516, 537, 524, 537, 519, 521, 482, 503, 484, 509, 515, 507, 478, 502, 510, 505, 534, 492, 504, 528, 520, 518, 537, 521, 535, 490, 507, 523, 525, 534, 487, 538, 504, 517, 500, 525, 504, 541, 511, 536, 507, 494, 475, 509, 492, 508] payload_rows_per_dst=[10972, 11040, 10858, 10904, 10990, 10948, 11054, 10858] dup_rows_per_dst=[5434, 5366, 5274, 5506, 5596, 5322, 5512, 5438] combine_rows_per_src=[10972, 11040, 10858, 10904, 10990, 10948, 11054, 10858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625164288 dispatch_meta_bytes=458856 combine_payload_bytes=625164288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625623144 ingress_bytes=700661760 link_bytes=700661760 token_bytes=1250787432 weight_movement_bytes=75497472 total_nvlink_bytes=1326284904 weight_to_token_ratio=0.060360 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.885377373e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624902144 dispatch_meta_bytes=457816 combine_payload_bytes=624902144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625359960 ingress_bytes=700399616 link_bytes=700399616 token_bytes=1250262104 weight_movement_bytes=75497472 total_nvlink_bytes=1325759576 weight_to_token_ratio=0.060385 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.882053018e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628703232 dispatch_meta_bytes=459216 combine_payload_bytes=628703232 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=629162448 ingress_bytes=855195648 link_bytes=855195648 token_bytes=1257865680 weight_movement_bytes=226492416 total_nvlink_bytes=1484358096 weight_to_token_ratio=0.180061 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084508460e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627212288 dispatch_meta_bytes=458816 combine_payload_bytes=627212288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627671104 ingress_bytes=702709760 link_bytes=702709760 token_bytes=1254883392 weight_movement_bytes=75497472 total_nvlink_bytes=1330380864 weight_to_token_ratio=0.060163 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911348896e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=459448 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627212984 ingress_bytes=702251008 link_bytes=702251008 token_bytes=1253966520 weight_movement_bytes=75497472 total_nvlink_bytes=1329463992 weight_to_token_ratio=0.060207 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905531274e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626819072 dispatch_meta_bytes=457568 combine_payload_bytes=626819072 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627276640 ingress_bytes=853311488 link_bytes=853311488 token_bytes=1254095712 weight_movement_bytes=226492416 total_nvlink_bytes=1480588128 weight_to_token_ratio=0.180602 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082119079e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=457944 combine_payload_bytes=627785728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628243672 ingress_bytes=703283200 link_bytes=703283200 token_bytes=1256029400 weight_movement_bytes=75497472 total_nvlink_bytes=1331526872 weight_to_token_ratio=0.060108 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918620922e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87520 payload_rows_remote=76508 dup_rows=43552 dup_rows_remote=37950 dedup_ratio=0.332275 router_tokens_per_expert=[499, 499, 518, 553, 520, 529, 473, 553, 518, 533, 479, 544, 516, 563, 524, 486, 506, 497, 520, 493, 505, 521, 511, 494, 520, 485, 503, 508, 535, 513, 580, 499, 499, 524, 539, 530, 496, 470, 527, 529, 528, 502, 508, 531, 519, 536, 508, 493, 511, 490, 536, 521, 494, 536, 509, 549, 514, 522, 501, 566, 507, 497, 484, 537, 478, 520, 521, 517, 524, 546, 522, 494, 509, 485, 489, 516, 480, 484, 515, 478, 501, 524, 520, 561, 510, 515, 496, 475, 504, 533, 509, 548, 493, 506, 510, 498, 514, 523, 514, 509, 491, 505, 490, 507, 499, 538, 503, 495, 524, 494, 523, 508, 489, 540, 508, 508, 526, 502, 528, 497, 484, 495, 511, 515, 492, 526, 471, 516] payload_rows_per_dst=[11012, 11008, 10876, 11088, 10844, 10998, 10774, 10920] dup_rows_per_dst=[5602, 5710, 5602, 5584, 5312, 5408, 5038, 5296] combine_rows_per_src=[11012, 11008, 10876, 11088, 10844, 10998, 10774, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87422 payload_rows_remote=76518 dup_rows=43650 dup_rows_remote=38296 dedup_ratio=0.333023 router_tokens_per_expert=[492, 482, 543, 503, 515, 500, 517, 477, 536, 494, 484, 554, 499, 526, 499, 477, 468, 507, 519, 501, 485, 510, 495, 481, 510, 498, 525, 544, 530, 507, 520, 529, 524, 479, 507, 517, 545, 528, 519, 497, 495, 522, 502, 524, 526, 483, 504, 523, 477, 510, 524, 515, 504, 522, 533, 483, 492, 507, 520, 483, 528, 516, 504, 568, 507, 553, 519, 516, 486, 536, 516, 492, 563, 498, 510, 480, 525, 513, 509, 517, 509, 494, 555, 565, 518, 532, 498, 505, 506, 488, 533, 530, 545, 504, 491, 513, 495, 520, 539, 506, 495, 506, 470, 526, 515, 536, 537, 538, 519, 536, 510, 522, 531, 493, 553, 485, 521, 459, 494, 511, 492, 551, 509, 519, 559, 499, 469, 487] payload_rows_per_dst=[10750, 10904, 10960, 10980, 10914, 11084, 10952, 10878] dup_rows_per_dst=[5446, 5354, 5430, 5392, 5566, 5488, 5588, 5386] combine_rows_per_src=[10750, 10904, 10960, 10980, 10914, 11084, 10952, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=457832 combine_payload_bytes=626753536 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627211368 ingress_bytes=853245952 link_bytes=853245952 token_bytes=1253964904 weight_movement_bytes=226492416 total_nvlink_bytes=1480457320 weight_to_token_ratio=0.180621 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082035971e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87224 payload_rows_remote=76250 dup_rows=43848 dup_rows_remote=38466 dedup_ratio=0.334534 router_tokens_per_expert=[528, 526, 531, 476, 495, 519, 525, 516, 489, 532, 525, 509, 530, 558, 538, 537, 532, 495, 526, 517, 534, 518, 524, 541, 565, 498, 519, 493, 530, 517, 535, 565, 526, 526, 489, 497, 500, 529, 515, 537, 524, 516, 516, 529, 509, 502, 467, 491, 484, 497, 511, 548, 481, 532, 507, 507, 514, 461, 478, 519, 470, 546, 501, 493, 501, 505, 539, 489, 527, 476, 543, 494, 484, 507, 513, 513, 510, 501, 512, 516, 541, 474, 518, 455, 486, 532, 515, 526, 460, 543, 520, 529, 548, 487, 540, 534, 461, 478, 530, 528, 467, 498, 497, 517, 531, 525, 507, 514, 548, 503, 538, 494, 552, 494, 510, 492, 496, 534, 509, 502, 458, 520, 515, 517, 519, 499, 487, 493] payload_rows_per_dst=[11068, 10614, 11154, 10858, 10860, 10758, 10938, 10974] dup_rows_per_dst=[5600, 5028, 5984, 5558, 5398, 5498, 5400, 5382] combine_rows_per_src=[11068, 10614, 11154, 10858, 10860, 10758, 10938, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87464 payload_rows_remote=76348 dup_rows=43608 dup_rows_remote=38002 dedup_ratio=0.332703 router_tokens_per_expert=[493, 553, 530, 507, 475, 524, 542, 499, 536, 560, 506, 532, 513, 471, 495, 502, 528, 534, 508, 493, 513, 521, 515, 470, 551, 506, 504, 477, 506, 497, 519, 518, 541, 503, 489, 509, 484, 484, 488, 512, 486, 497, 523, 517, 494, 528, 518, 504, 510, 521, 504, 514, 510, 512, 491, 520, 499, 537, 467, 517, 513, 488, 466, 558, 509, 503, 508, 499, 504, 503, 507, 513, 547, 497, 525, 487, 520, 500, 496, 529, 548, 527, 487, 544, 495, 544, 517, 532, 491, 522, 521, 486, 528, 510, 531, 502, 485, 514, 524, 519, 524, 557, 505, 522, 548, 526, 533, 551, 519, 537, 474, 523, 489, 493, 519, 476, 535, 492, 519, 501, 557, 497, 495, 497, 499, 529, 562, 481] payload_rows_per_dst=[10990, 10912, 10776, 10866, 10906, 11014, 11116, 10884] dup_rows_per_dst=[5486, 5408, 5378, 5388, 5388, 5556, 5606, 5398] combine_rows_per_src=[10990, 10912, 10776, 10866, 10906, 11014, 11116, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87344 payload_rows_remote=76216 dup_rows=43728 dup_rows_remote=38206 dedup_ratio=0.333618 router_tokens_per_expert=[465, 497, 522, 454, 504, 529, 519, 514, 533, 476, 513, 527, 506, 530, 547, 502, 540, 475, 518, 498, 571, 521, 542, 522, 531, 494, 506, 495, 514, 519, 508, 539, 506, 491, 469, 475, 498, 518, 502, 522, 488, 520, 511, 534, 510, 528, 544, 523, 496, 537, 498, 491, 481, 516, 512, 504, 527, 504, 504, 520, 506, 524, 451, 507, 533, 541, 506, 539, 509, 492, 540, 485, 532, 501, 518, 503, 478, 534, 548, 523, 535, 521, 539, 492, 541, 533, 525, 541, 538, 523, 476, 528, 517, 530, 490, 496, 503, 527, 467, 544, 517, 516, 487, 498, 500, 530, 491, 537, 535, 527, 513, 495, 543, 525, 490, 496, 528, 471, 517, 486, 526, 484, 493, 553, 519, 488, 484, 491] payload_rows_per_dst=[10854, 11084, 10826, 10842, 10966, 11128, 10824, 10820] dup_rows_per_dst=[5422, 5502, 5452, 5314, 5598, 5522, 5550, 5368] combine_rows_per_src=[10854, 11084, 10826, 10842, 10966, 11128, 10824, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87216 payload_rows_remote=76380 dup_rows=43856 dup_rows_remote=38394 dedup_ratio=0.334595 router_tokens_per_expert=[530, 554, 491, 537, 509, 470, 533, 518, 496, 526, 501, 512, 482, 522, 536, 526, 524, 498, 489, 563, 541, 548, 497, 524, 560, 452, 492, 538, 490, 537, 491, 498, 509, 518, 547, 537, 507, 537, 503, 507, 476, 481, 511, 513, 499, 509, 507, 488, 494, 491, 479, 536, 505, 516, 516, 530, 543, 500, 529, 504, 502, 534, 503, 519, 463, 523, 532, 518, 555, 505, 493, 468, 506, 500, 495, 523, 534, 528, 506, 491, 539, 547, 511, 484, 477, 514, 547, 477, 535, 525, 487, 477, 524, 487, 536, 523, 495, 511, 524, 516, 471, 531, 521, 518, 492, 550, 476, 497, 520, 485, 500, 518, 530, 532, 501, 505, 493, 534, 505, 522, 540, 528, 484, 536, 492, 527, 510, 507] payload_rows_per_dst=[11038, 10938, 10836, 10904, 10786, 10986, 10830, 10898] dup_rows_per_dst=[5448, 5546, 5462, 5498, 5494, 5394, 5420, 5594] combine_rows_per_src=[11038, 10938, 10836, 10904, 10786, 10986, 10830, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626835456 dispatch_meta_bytes=459256 combine_payload_bytes=626835456 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627294712 ingress_bytes=928825344 link_bytes=928825344 token_bytes=1254130168 weight_movement_bytes=301989888 total_nvlink_bytes=1556120056 weight_to_token_ratio=0.240796 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177881278e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87634 payload_rows_remote=76634 dup_rows=43438 dup_rows_remote=38016 dedup_ratio=0.331406 router_tokens_per_expert=[510, 522, 540, 499, 547, 506, 500, 479, 510, 507, 528, 535, 483, 504, 515, 480, 523, 483, 532, 516, 506, 504, 514, 494, 493, 533, 556, 530, 543, 512, 540, 484, 537, 505, 514, 496, 504, 499, 511, 512, 520, 499, 484, 513, 537, 487, 507, 496, 530, 491, 530, 513, 522, 527, 520, 521, 520, 527, 470, 506, 505, 547, 548, 527, 513, 519, 519, 542, 491, 506, 528, 515, 508, 496, 500, 491, 529, 514, 513, 527, 526, 514, 477, 495, 497, 530, 477, 487, 466, 513, 501, 499, 494, 502, 498, 499, 524, 523, 527, 494, 487, 493, 536, 544, 507, 555, 518, 532, 524, 513, 474, 513, 473, 533, 506, 532, 522, 516, 503, 493, 513, 514, 532, 511, 506, 537, 532, 510] payload_rows_per_dst=[10944, 10938, 10850, 11084, 11000, 10694, 11060, 11064] dup_rows_per_dst=[5386, 5588, 5392, 5524, 5422, 5256, 5468, 5402] combine_rows_per_src=[10944, 10938, 10850, 11084, 11000, 10694, 11060, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87712 payload_rows_remote=76678 dup_rows=43360 dup_rows_remote=37850 dedup_ratio=0.330811 router_tokens_per_expert=[511, 504, 517, 518, 484, 486, 543, 524, 510, 498, 489, 537, 484, 466, 519, 531, 527, 520, 501, 516, 522, 515, 544, 499, 504, 517, 491, 516, 525, 492, 517, 543, 489, 495, 517, 507, 511, 510, 511, 486, 499, 497, 517, 525, 528, 505, 533, 498, 498, 552, 517, 510, 486, 500, 554, 520, 544, 479, 478, 515, 546, 556, 509, 508, 506, 517, 522, 509, 515, 535, 519, 481, 516, 540, 495, 505, 498, 520, 531, 530, 496, 511, 511, 533, 503, 502, 502, 518, 501, 480, 515, 514, 551, 507, 528, 502, 448, 522, 503, 533, 499, 494, 486, 538, 500, 480, 547, 507, 527, 499, 489, 554, 499, 534, 512, 485, 525, 532, 528, 503, 537, 516, 523, 525, 486, 482, 490, 550] payload_rows_per_dst=[10912, 11056, 10936, 11034, 10934, 10948, 10938, 10954] dup_rows_per_dst=[5330, 5442, 5320, 5510, 5544, 5400, 5314, 5500] combine_rows_per_src=[10912, 11056, 10936, 11034, 10934, 10948, 10938, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=624640000 dispatch_meta_bytes=458864 combine_payload_bytes=624640000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625098864 ingress_bytes=700137472 link_bytes=700137472 token_bytes=1249738864 weight_movement_bytes=75497472 total_nvlink_bytes=1325236336 weight_to_token_ratio=0.060411 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.878728663e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625442816 dispatch_meta_bytes=457400 combine_payload_bytes=625442816 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625900216 ingress_bytes=700940288 link_bytes=700940288 token_bytes=1251343032 weight_movement_bytes=75497472 total_nvlink_bytes=1326840504 weight_to_token_ratio=0.060333 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888909500e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=624361472 dispatch_meta_bytes=457688 combine_payload_bytes=624361472 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624819160 ingress_bytes=775356416 link_bytes=775356416 token_bytes=1249180632 weight_movement_bytes=150994944 total_nvlink_bytes=1400175576 weight_to_token_ratio=0.120875 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.832610752e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625704960 dispatch_meta_bytes=459096 combine_payload_bytes=625704960 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626164056 ingress_bytes=701202432 link_bytes=701202432 token_bytes=1251869016 weight_movement_bytes=75497472 total_nvlink_bytes=1327366488 weight_to_token_ratio=0.060308 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892233855e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=458600 combine_payload_bytes=627785728 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628244328 ingress_bytes=854278144 link_bytes=854278144 token_bytes=1256030056 weight_movement_bytes=226492416 total_nvlink_bytes=1482522472 weight_to_token_ratio=0.180324 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083344935e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628146176 dispatch_meta_bytes=458112 combine_payload_bytes=628146176 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628604288 ingress_bytes=703643648 link_bytes=703643648 token_bytes=1256750464 weight_movement_bytes=75497472 total_nvlink_bytes=1332247936 weight_to_token_ratio=0.060074 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923191910e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87436 payload_rows_remote=76462 dup_rows=43636 dup_rows_remote=38212 dedup_ratio=0.332916 router_tokens_per_expert=[514, 535, 477, 567, 486, 515, 536, 510, 517, 488, 511, 513, 488, 558, 492, 486, 524, 508, 508, 510, 552, 538, 543, 525, 522, 532, 513, 536, 520, 530, 470, 545, 521, 476, 530, 537, 514, 525, 492, 467, 518, 509, 506, 499, 538, 496, 507, 523, 500, 483, 495, 524, 504, 499, 509, 498, 534, 529, 507, 543, 500, 543, 524, 468, 492, 567, 513, 528, 495, 495, 480, 521, 529, 509, 490, 508, 520, 533, 493, 484, 510, 508, 509, 526, 485, 506, 508, 500, 511, 494, 512, 552, 519, 492, 516, 525, 519, 512, 518, 445, 500, 533, 490, 518, 518, 498, 479, 478, 511, 503, 492, 520, 504, 507, 553, 527, 496, 517, 511, 510, 521, 537, 504, 528, 506, 522, 519, 523] payload_rows_per_dst=[10974, 11112, 10848, 10872, 10934, 10960, 10744, 10992] dup_rows_per_dst=[5424, 5646, 5468, 5448, 5362, 5386, 5324, 5578] combine_rows_per_src=[10974, 11112, 10848, 10872, 10934, 10960, 10744, 10992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76588 dup_rows=43514 dup_rows_remote=37916 dedup_ratio=0.331985 router_tokens_per_expert=[504, 510, 513, 532, 506, 538, 481, 498, 542, 509, 468, 494, 532, 499, 493, 463, 539, 525, 522, 503, 503, 510, 505, 504, 503, 531, 547, 520, 525, 516, 508, 523, 534, 507, 517, 525, 470, 502, 487, 500, 542, 495, 515, 493, 511, 517, 555, 477, 536, 497, 495, 516, 529, 498, 507, 459, 503, 503, 514, 533, 518, 526, 491, 519, 550, 493, 494, 521, 552, 533, 499, 502, 521, 498, 510, 506, 521, 551, 492, 540, 491, 512, 518, 479, 519, 510, 501, 525, 564, 513, 512, 528, 492, 497, 530, 496, 519, 510, 491, 514, 526, 498, 498, 504, 502, 512, 509, 519, 487, 542, 520, 496, 483, 511, 523, 493, 501, 551, 479, 529, 499, 515, 539, 520, 537, 515, 547, 520] payload_rows_per_dst=[10874, 10970, 10940, 10950, 11008, 10982, 10942, 10892] dup_rows_per_dst=[5290, 5598, 5354, 5338, 5558, 5392, 5352, 5632] combine_rows_per_src=[10874, 10970, 10940, 10950, 11008, 10982, 10942, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87814 payload_rows_remote=76622 dup_rows=43258 dup_rows_remote=37346 dedup_ratio=0.330032 router_tokens_per_expert=[546, 507, 541, 514, 491, 492, 522, 524, 544, 494, 513, 487, 498, 510, 512, 500, 517, 524, 530, 521, 509, 521, 524, 521, 534, 543, 498, 476, 479, 536, 559, 538, 532, 479, 481, 520, 487, 487, 526, 526, 502, 497, 517, 495, 528, 550, 499, 484, 519, 531, 447, 498, 515, 526, 532, 523, 520, 518, 552, 518, 513, 499, 490, 503, 506, 517, 514, 489, 560, 465, 501, 482, 478, 518, 520, 530, 489, 543, 470, 528, 517, 515, 496, 494, 528, 475, 508, 482, 534, 512, 515, 517, 518, 556, 521, 521, 526, 479, 494, 540, 494, 505, 510, 508, 510, 519, 503, 483, 519, 475, 529, 514, 518, 540, 540, 522, 508, 543, 477, 528, 522, 502, 491, 488, 501, 553, 511, 526] payload_rows_per_dst=[11036, 10848, 10816, 10944, 10950, 11038, 10990, 11192] dup_rows_per_dst=[5354, 5248, 5292, 5140, 5270, 5492, 5550, 5912] combine_rows_per_src=[11036, 10848, 10816, 10944, 10950, 11038, 10990, 11192] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626376704 dispatch_meta_bytes=458696 combine_payload_bytes=626376704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626835400 ingress_bytes=701874176 link_bytes=701874176 token_bytes=1253212104 weight_movement_bytes=75497472 total_nvlink_bytes=1328709576 weight_to_token_ratio=0.060243 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900752514e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87776 payload_rows_remote=76956 dup_rows=43296 dup_rows_remote=37894 dedup_ratio=0.330322 router_tokens_per_expert=[472, 514, 527, 488, 505, 504, 486, 500, 528, 497, 542, 506, 546, 508, 506, 524, 554, 524, 505, 527, 556, 500, 549, 511, 493, 508, 510, 490, 504, 542, 559, 540, 537, 497, 536, 522, 514, 511, 535, 514, 516, 493, 543, 520, 525, 497, 524, 489, 548, 521, 528, 509, 516, 534, 545, 513, 487, 512, 521, 497, 502, 478, 502, 468, 499, 510, 510, 515, 474, 535, 527, 523, 523, 510, 530, 492, 491, 504, 526, 516, 511, 513, 552, 520, 538, 511, 478, 553, 490, 552, 493, 498, 501, 514, 492, 464, 510, 523, 495, 512, 499, 463, 514, 496, 490, 525, 533, 508, 515, 500, 518, 510, 522, 485, 510, 510, 511, 531, 502, 492, 499, 508, 500, 492, 525, 520, 487, 487] payload_rows_per_dst=[10976, 11170, 11010, 10952, 11028, 10982, 10820, 10838] dup_rows_per_dst=[5330, 5574, 5536, 5410, 5342, 5378, 5402, 5324] combine_rows_per_src=[10976, 11170, 11010, 10952, 11028, 10982, 10820, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87664 payload_rows_remote=76582 dup_rows=43408 dup_rows_remote=37594 dedup_ratio=0.331177 router_tokens_per_expert=[539, 542, 469, 525, 494, 508, 458, 537, 488, 524, 480, 511, 485, 477, 483, 518, 487, 516, 507, 480, 571, 514, 488, 505, 514, 500, 474, 500, 504, 536, 469, 513, 522, 534, 507, 470, 518, 528, 536, 517, 488, 523, 495, 484, 509, 533, 529, 502, 510, 544, 507, 534, 516, 508, 532, 505, 506, 527, 529, 487, 494, 532, 525, 486, 523, 487, 496, 509, 519, 519, 521, 539, 519, 549, 515, 479, 480, 476, 483, 514, 556, 535, 522, 509, 510, 542, 517, 530, 532, 529, 513, 511, 548, 534, 498, 562, 516, 517, 506, 526, 518, 511, 539, 522, 515, 502, 512, 513, 548, 503, 518, 498, 500, 501, 526, 517, 538, 503, 500, 531, 537, 474, 497, 520, 480, 518, 485, 516] payload_rows_per_dst=[10842, 10840, 11058, 11022, 10898, 11082, 11068, 10854] dup_rows_per_dst=[5234, 5316, 5332, 5462, 5358, 5814, 5460, 5432] combine_rows_per_src=[10842, 10840, 11058, 11022, 10898, 11082, 11068, 10854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87432 payload_rows_remote=76498 dup_rows=43640 dup_rows_remote=38168 dedup_ratio=0.332947 router_tokens_per_expert=[506, 503, 480, 531, 496, 495, 492, 490, 491, 513, 538, 509, 476, 478, 540, 524, 550, 549, 514, 482, 492, 538, 512, 512, 490, 511, 537, 505, 493, 497, 530, 496, 541, 482, 510, 495, 473, 538, 542, 532, 548, 510, 496, 507, 510, 494, 525, 500, 551, 512, 532, 540, 498, 528, 504, 543, 500, 493, 531, 497, 525, 518, 504, 524, 531, 543, 517, 516, 500, 479, 490, 508, 517, 516, 502, 508, 547, 523, 533, 530, 531, 513, 524, 447, 504, 524, 496, 482, 510, 483, 455, 535, 525, 537, 549, 543, 489, 521, 526, 502, 484, 519, 511, 531, 533, 503, 502, 501, 502, 496, 502, 517, 513, 491, 525, 535, 478, 524, 518, 512, 482, 510, 533, 486, 522, 542, 490, 545] payload_rows_per_dst=[10840, 10922, 10934, 10936, 10914, 10922, 10956, 11008] dup_rows_per_dst=[5284, 5494, 5472, 5664, 5606, 5394, 5322, 5404] combine_rows_per_src=[10840, 10922, 10934, 10936, 10914, 10922, 10956, 11008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627408896 dispatch_meta_bytes=458016 combine_payload_bytes=627408896 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627866912 ingress_bytes=853901312 link_bytes=853901312 token_bytes=1255275808 weight_movement_bytes=226492416 total_nvlink_bytes=1481768224 weight_to_token_ratio=0.180432 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082867059e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627687424 dispatch_meta_bytes=455872 combine_payload_bytes=627687424 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628143296 ingress_bytes=703184896 link_bytes=703184896 token_bytes=1255830720 weight_movement_bytes=75497472 total_nvlink_bytes=1331328192 weight_to_token_ratio=0.060118 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917374289e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87326 payload_rows_remote=76440 dup_rows=43746 dup_rows_remote=38330 dedup_ratio=0.333755 router_tokens_per_expert=[473, 493, 525, 476, 513, 493, 507, 531, 473, 515, 502, 495, 529, 517, 474, 521, 503, 526, 516, 494, 508, 493, 511, 512, 501, 509, 540, 556, 541, 504, 514, 503, 539, 517, 532, 543, 547, 500, 487, 491, 540, 493, 545, 480, 524, 481, 539, 520, 487, 491, 521, 510, 475, 520, 548, 528, 534, 529, 541, 481, 515, 482, 511, 546, 511, 528, 525, 540, 471, 535, 498, 495, 487, 494, 505, 533, 530, 521, 458, 520, 500, 521, 497, 552, 520, 521, 526, 492, 486, 494, 497, 549, 492, 519, 558, 504, 532, 535, 528, 507, 508, 544, 523, 484, 536, 523, 522, 484, 498, 513, 492, 463, 501, 501, 527, 555, 509, 529, 511, 525, 472, 482, 512, 559, 492, 490, 538, 497] payload_rows_per_dst=[10818, 11030, 10956, 10968, 10886, 10928, 10860, 10880] dup_rows_per_dst=[5256, 5432, 5600, 5470, 5416, 5528, 5524, 5520] combine_rows_per_src=[10818, 11030, 10956, 10968, 10886, 10928, 10860, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87516 payload_rows_remote=76700 dup_rows=43556 dup_rows_remote=38048 dedup_ratio=0.332306 router_tokens_per_expert=[466, 516, 494, 495, 545, 548, 524, 544, 550, 540, 525, 538, 507, 494, 481, 528, 506, 516, 514, 552, 480, 498, 523, 551, 554, 512, 477, 498, 480, 534, 542, 539, 550, 513, 524, 524, 458, 535, 512, 529, 515, 499, 541, 501, 473, 484, 539, 507, 511, 513, 529, 506, 510, 514, 505, 468, 494, 529, 496, 500, 550, 510, 542, 485, 534, 491, 536, 509, 497, 497, 538, 502, 526, 493, 522, 561, 488, 513, 503, 517, 473, 548, 527, 499, 526, 507, 510, 521, 512, 498, 512, 528, 506, 489, 541, 501, 520, 501, 467, 512, 519, 555, 509, 470, 534, 518, 519, 533, 473, 535, 511, 514, 526, 513, 508, 466, 503, 516, 496, 469, 494, 509, 524, 532, 500, 457, 474, 497] payload_rows_per_dst=[10992, 11092, 10946, 10816, 10972, 10910, 10964, 10824] dup_rows_per_dst=[5598, 5460, 5462, 5508, 5482, 5486, 5416, 5144] combine_rows_per_src=[10992, 11092, 10946, 10816, 10972, 10910, 10964, 10824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=630423552 dispatch_meta_bytes=459400 combine_payload_bytes=630423552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630882952 ingress_bytes=705921024 link_bytes=705921024 token_bytes=1261306504 weight_movement_bytes=75497472 total_nvlink_bytes=1336803976 weight_to_token_ratio=0.059857 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.952072243e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627359744 dispatch_meta_bytes=456704 combine_payload_bytes=627359744 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627816448 ingress_bytes=778354688 link_bytes=778354688 token_bytes=1255176192 weight_movement_bytes=150994944 total_nvlink_bytes=1406171136 weight_to_token_ratio=0.120298 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.870633061e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626671616 dispatch_meta_bytes=458664 combine_payload_bytes=626671616 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627130280 ingress_bytes=777666560 link_bytes=777666560 token_bytes=1253801896 weight_movement_bytes=150994944 total_nvlink_bytes=1404796840 weight_to_token_ratio=0.120430 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861906629e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626196480 dispatch_meta_bytes=459080 combine_payload_bytes=626196480 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=626655560 ingress_bytes=777191424 link_bytes=777191424 token_bytes=1252852040 weight_movement_bytes=150994944 total_nvlink_bytes=1403846984 weight_to_token_ratio=0.120521 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.855881236e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628326400 dispatch_meta_bytes=458992 combine_payload_bytes=628326400 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628785392 ingress_bytes=779321344 link_bytes=779321344 token_bytes=1257111792 weight_movement_bytes=150994944 total_nvlink_bytes=1408106736 weight_to_token_ratio=0.120113 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.882891620e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87422 payload_rows_remote=76122 dup_rows=43650 dup_rows_remote=37602 dedup_ratio=0.333023 router_tokens_per_expert=[463, 527, 521, 541, 513, 507, 533, 534, 538, 487, 466, 502, 528, 529, 497, 487, 513, 535, 540, 558, 514, 519, 519, 503, 551, 511, 508, 519, 513, 526, 541, 535, 493, 482, 507, 496, 515, 516, 528, 507, 521, 519, 516, 513, 539, 510, 494, 516, 552, 446, 527, 479, 498, 503, 541, 514, 516, 523, 525, 493, 506, 524, 562, 518, 485, 499, 515, 526, 543, 524, 470, 527, 516, 516, 575, 470, 535, 485, 531, 504, 519, 488, 516, 508, 498, 490, 542, 516, 480, 517, 466, 495, 478, 482, 504, 508, 493, 517, 534, 520, 476, 503, 525, 520, 544, 521, 500, 528, 542, 497, 526, 477, 571, 507, 501, 464, 506, 507, 519, 539, 497, 498, 516, 500, 482, 491, 496, 514] payload_rows_per_dst=[11300, 10840, 10992, 10914, 10944, 10720, 10926, 10786] dup_rows_per_dst=[6048, 5238, 5372, 5578, 5440, 5024, 5520, 5430] combine_rows_per_src=[11300, 10840, 10992, 10914, 10944, 10720, 10926, 10786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=86988 payload_rows_remote=76184 dup_rows=44084 dup_rows_remote=38742 dedup_ratio=0.336334 router_tokens_per_expert=[512, 497, 485, 505, 484, 499, 477, 494, 557, 543, 487, 511, 503, 558, 469, 508, 531, 490, 494, 542, 474, 460, 511, 509, 513, 510, 538, 469, 546, 513, 493, 480, 472, 502, 558, 533, 543, 519, 505, 525, 514, 549, 517, 486, 487, 532, 531, 523, 491, 485, 493, 527, 495, 517, 477, 493, 571, 515, 461, 493, 523, 536, 511, 517, 496, 514, 501, 539, 534, 521, 556, 497, 541, 551, 525, 504, 512, 503, 497, 525, 536, 490, 551, 531, 519, 524, 508, 490, 517, 494, 501, 509, 498, 494, 498, 508, 505, 540, 496, 544, 494, 474, 519, 485, 540, 533, 523, 526, 467, 513, 530, 524, 487, 530, 519, 516, 504, 517, 535, 536, 493, 482, 491, 537, 541, 526, 526, 536] payload_rows_per_dst=[10744, 10804, 11046, 10784, 10988, 10904, 10816, 10902] dup_rows_per_dst=[5434, 5342, 5546, 5426, 5644, 5432, 5610, 5650] combine_rows_per_src=[10744, 10804, 11046, 10784, 10988, 10904, 10816, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76424 dup_rows=43570 dup_rows_remote=38090 dedup_ratio=0.332413 router_tokens_per_expert=[523, 531, 493, 534, 492, 529, 494, 479, 516, 537, 497, 536, 482, 511, 515, 524, 462, 474, 506, 492, 522, 510, 529, 525, 504, 508, 509, 498, 500, 517, 558, 481, 509, 495, 532, 519, 520, 522, 491, 522, 513, 502, 562, 521, 472, 521, 498, 509, 504, 486, 485, 503, 526, 520, 505, 525, 511, 499, 506, 512, 537, 505, 546, 525, 484, 488, 533, 545, 496, 544, 487, 511, 535, 532, 491, 494, 512, 524, 516, 471, 499, 541, 480, 501, 499, 496, 538, 518, 544, 500, 522, 473, 531, 529, 525, 500, 510, 547, 549, 511, 506, 494, 519, 535, 497, 493, 479, 503, 503, 531, 498, 532, 525, 509, 494, 510, 543, 539, 490, 529, 536, 575, 518, 531, 506, 495, 475, 504] payload_rows_per_dst=[11016, 10908, 10504, 11388, 10836, 10814, 10958, 11078] dup_rows_per_dst=[5370, 5282, 4868, 6046, 5408, 5578, 5538, 5480] combine_rows_per_src=[11016, 10908, 10504, 11388, 10836, 10814, 10958, 11078] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87530 payload_rows_remote=76596 dup_rows=43542 dup_rows_remote=38116 dedup_ratio=0.332199 router_tokens_per_expert=[528, 509, 517, 478, 475, 514, 519, 550, 529, 504, 496, 477, 487, 510, 538, 502, 493, 520, 524, 511, 499, 551, 521, 560, 490, 499, 525, 513, 510, 513, 514, 504, 497, 491, 493, 561, 520, 516, 525, 510, 521, 509, 477, 505, 494, 526, 523, 503, 490, 487, 505, 508, 484, 511, 530, 506, 534, 536, 502, 516, 511, 469, 514, 529, 516, 498, 513, 486, 520, 529, 494, 511, 549, 530, 513, 507, 545, 515, 557, 498, 504, 508, 502, 473, 523, 464, 498, 504, 522, 529, 523, 523, 498, 548, 505, 550, 512, 513, 509, 507, 532, 516, 536, 529, 492, 516, 537, 503, 504, 491, 480, 503, 504, 533, 524, 513, 499, 520, 521, 511, 485, 531, 535, 498, 533, 500, 523, 488] payload_rows_per_dst=[10882, 10960, 10892, 10884, 11086, 10922, 10934, 10970] dup_rows_per_dst=[5384, 5534, 5436, 5394, 5476, 5426, 5426, 5466] combine_rows_per_src=[10882, 10960, 10892, 10884, 11086, 10922, 10934, 10970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=623591424 dispatch_meta_bytes=454896 combine_payload_bytes=623591424 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624046320 ingress_bytes=699088896 link_bytes=699088896 token_bytes=1247637744 weight_movement_bytes=75497472 total_nvlink_bytes=1323135216 weight_to_token_ratio=0.060512 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.865431243e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87492 payload_rows_remote=76636 dup_rows=43580 dup_rows_remote=38140 dedup_ratio=0.332489 router_tokens_per_expert=[505, 489, 519, 525, 522, 481, 474, 533, 539, 540, 498, 539, 521, 529, 516, 490, 510, 542, 503, 506, 511, 478, 494, 477, 532, 482, 539, 487, 537, 527, 539, 512, 491, 523, 520, 513, 531, 522, 497, 498, 483, 510, 554, 540, 505, 521, 519, 528, 496, 501, 523, 545, 482, 493, 492, 510, 489, 496, 481, 530, 505, 514, 480, 520, 506, 546, 497, 504, 541, 481, 491, 493, 518, 489, 464, 564, 529, 514, 522, 478, 518, 514, 490, 519, 501, 499, 525, 518, 503, 483, 511, 524, 501, 524, 516, 502, 520, 499, 505, 551, 550, 531, 502, 503, 530, 551, 524, 573, 510, 508, 498, 546, 575, 513, 459, 524, 528, 510, 510, 534, 507, 475, 498, 476, 532, 463, 532, 506] payload_rows_per_dst=[10964, 11032, 10956, 10728, 10856, 10856, 11098, 11002] dup_rows_per_dst=[5476, 5320, 5554, 5386, 5418, 5440, 5704, 5282] combine_rows_per_src=[10964, 11032, 10956, 10728, 10856, 10856, 11098, 11002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87310 payload_rows_remote=76382 dup_rows=43762 dup_rows_remote=38326 dedup_ratio=0.333878 router_tokens_per_expert=[515, 539, 498, 473, 489, 552, 482, 517, 518, 536, 480, 531, 498, 510, 567, 483, 518, 526, 510, 516, 511, 503, 485, 499, 527, 509, 509, 513, 506, 536, 555, 513, 485, 477, 530, 501, 531, 526, 519, 547, 514, 535, 498, 502, 487, 474, 521, 535, 533, 531, 502, 504, 536, 493, 517, 523, 505, 487, 537, 547, 512, 495, 485, 477, 499, 512, 514, 521, 513, 500, 513, 519, 513, 529, 514, 482, 492, 527, 516, 506, 519, 499, 510, 469, 515, 552, 517, 492, 527, 495, 524, 518, 545, 485, 502, 475, 497, 535, 516, 503, 515, 506, 503, 503, 551, 514, 551, 512, 526, 538, 510, 521, 489, 488, 468, 538, 504, 519, 490, 470, 488, 532, 510, 518, 553, 519, 516, 529] payload_rows_per_dst=[10876, 10914, 10928, 10948, 10884, 10848, 10966, 10946] dup_rows_per_dst=[5500, 5558, 5436, 5420, 5456, 5440, 5636, 5316] combine_rows_per_src=[10876, 10914, 10928, 10948, 10884, 10848, 10966, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=624099328 dispatch_meta_bytes=459704 combine_payload_bytes=624099328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624559032 ingress_bytes=699596800 link_bytes=699596800 token_bytes=1248658360 weight_movement_bytes=75497472 total_nvlink_bytes=1324155832 weight_to_token_ratio=0.060463 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.871872181e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=458056 combine_payload_bytes=626065408 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626523464 ingress_bytes=852557824 link_bytes=852557824 token_bytes=1252588872 weight_movement_bytes=226492416 total_nvlink_bytes=1479081288 weight_to_token_ratio=0.180819 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081163327e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87616 payload_rows_remote=76632 dup_rows=43456 dup_rows_remote=38042 dedup_ratio=0.331543 router_tokens_per_expert=[505, 529, 542, 526, 525, 537, 522, 522, 477, 493, 555, 540, 498, 500, 529, 494, 514, 518, 502, 451, 520, 498, 492, 518, 558, 476, 580, 540, 538, 534, 490, 462, 512, 476, 487, 525, 528, 541, 518, 513, 536, 523, 514, 505, 525, 544, 521, 524, 501, 497, 531, 535, 499, 522, 520, 488, 497, 514, 524, 513, 513, 483, 530, 501, 515, 494, 534, 513, 494, 532, 514, 529, 505, 532, 503, 484, 506, 532, 465, 547, 508, 546, 504, 479, 509, 517, 453, 481, 501, 511, 539, 526, 498, 500, 484, 507, 519, 527, 521, 545, 497, 502, 501, 513, 513, 504, 498, 538, 510, 483, 511, 521, 509, 500, 523, 479, 518, 514, 497, 545, 524, 483, 509, 484, 515, 510, 502, 514] payload_rows_per_dst=[11008, 10932, 11118, 10938, 10984, 10844, 11008, 10784] dup_rows_per_dst=[5580, 5450, 5466, 5398, 5414, 5282, 5398, 5468] combine_rows_per_src=[11008, 10932, 11118, 10938, 10984, 10844, 11008, 10784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87436 payload_rows_remote=76594 dup_rows=43636 dup_rows_remote=38214 dedup_ratio=0.332916 router_tokens_per_expert=[531, 496, 483, 545, 499, 505, 557, 467, 476, 519, 508, 546, 476, 493, 522, 528, 528, 493, 527, 516, 520, 514, 461, 523, 509, 497, 499, 476, 507, 534, 514, 533, 529, 540, 536, 497, 496, 555, 566, 500, 499, 551, 495, 485, 510, 532, 509, 515, 485, 495, 506, 516, 543, 504, 512, 495, 498, 482, 540, 491, 533, 513, 494, 525, 539, 491, 534, 522, 513, 520, 540, 502, 496, 495, 509, 521, 477, 488, 495, 475, 515, 486, 503, 478, 503, 539, 540, 495, 497, 536, 511, 503, 507, 523, 546, 497, 499, 522, 509, 506, 512, 457, 531, 491, 515, 486, 492, 511, 522, 521, 508, 496, 498, 501, 531, 550, 519, 548, 548, 517, 538, 548, 540, 502, 509, 517, 552, 495] payload_rows_per_dst=[11002, 10890, 11046, 10842, 10852, 10872, 10838, 11094] dup_rows_per_dst=[5300, 5412, 5584, 5422, 5382, 5486, 5318, 5732] combine_rows_per_src=[11002, 10890, 11046, 10842, 10852, 10872, 10838, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627474432 dispatch_meta_bytes=458848 combine_payload_bytes=627474432 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627933280 ingress_bytes=778469376 link_bytes=778469376 token_bytes=1255407712 weight_movement_bytes=150994944 total_nvlink_bytes=1406402656 weight_to_token_ratio=0.120276 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872087466e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=459104 combine_payload_bytes=627802112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628261216 ingress_bytes=703299584 link_bytes=703299584 token_bytes=1256063328 weight_movement_bytes=75497472 total_nvlink_bytes=1331560800 weight_to_token_ratio=0.060106 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918828694e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625721344 dispatch_meta_bytes=458832 combine_payload_bytes=625721344 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626180176 ingress_bytes=776716288 link_bytes=776716288 token_bytes=1251901520 weight_movement_bytes=150994944 total_nvlink_bytes=1402896464 weight_to_token_ratio=0.120612 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.849855843e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=458696 combine_payload_bytes=627769344 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628228040 ingress_bytes=854261760 link_bytes=854261760 token_bytes=1255997384 weight_movement_bytes=226492416 total_nvlink_bytes=1482489800 weight_to_token_ratio=0.180329 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083324158e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627458048 dispatch_meta_bytes=459232 combine_payload_bytes=627458048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627917280 ingress_bytes=702955520 link_bytes=702955520 token_bytes=1255375328 weight_movement_bytes=75497472 total_nvlink_bytes=1330872800 weight_to_token_ratio=0.060139 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914465478e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87472 payload_rows_remote=76542 dup_rows=43600 dup_rows_remote=37994 dedup_ratio=0.332642 router_tokens_per_expert=[506, 586, 508, 500, 510, 498, 542, 510, 526, 531, 467, 515, 514, 499, 462, 537, 467, 478, 532, 515, 549, 514, 529, 513, 490, 566, 484, 533, 509, 549, 549, 496, 503, 516, 532, 545, 510, 520, 533, 539, 520, 510, 493, 496, 495, 488, 521, 524, 502, 505, 496, 508, 517, 513, 495, 487, 532, 464, 491, 479, 533, 496, 502, 514, 494, 508, 540, 510, 503, 505, 502, 520, 503, 539, 557, 507, 541, 529, 482, 545, 533, 522, 487, 524, 507, 519, 506, 503, 521, 470, 492, 529, 524, 523, 489, 528, 534, 528, 517, 524, 505, 498, 487, 479, 527, 495, 505, 523, 508, 496, 535, 538, 493, 484, 505, 513, 520, 506, 518, 505, 539, 483, 513, 485, 500, 509, 521, 518] payload_rows_per_dst=[10930, 11174, 11044, 10744, 10884, 11102, 10780, 10814] dup_rows_per_dst=[5606, 5560, 5648, 5324, 5504, 5826, 5044, 5088] combine_rows_per_src=[10930, 11174, 11044, 10744, 10884, 11102, 10780, 10814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87598 payload_rows_remote=76276 dup_rows=43474 dup_rows_remote=37404 dedup_ratio=0.331680 router_tokens_per_expert=[486, 543, 496, 483, 513, 523, 487, 535, 499, 497, 513, 490, 499, 518, 539, 530, 496, 504, 544, 540, 514, 521, 492, 508, 502, 497, 505, 482, 523, 559, 494, 493, 540, 520, 490, 505, 487, 501, 487, 504, 536, 543, 510, 505, 507, 513, 474, 521, 510, 551, 507, 518, 501, 498, 514, 539, 518, 507, 503, 560, 507, 497, 507, 507, 484, 493, 512, 495, 479, 546, 505, 500, 545, 545, 505, 496, 542, 503, 547, 495, 508, 541, 541, 538, 540, 525, 512, 513, 520, 487, 519, 506, 494, 498, 542, 492, 507, 532, 497, 514, 473, 508, 519, 513, 507, 494, 497, 482, 489, 520, 519, 498, 493, 518, 487, 552, 522, 526, 472, 525, 497, 520, 523, 548, 522, 525, 562, 495] payload_rows_per_dst=[10480, 10928, 10904, 11068, 10646, 11056, 11194, 11322] dup_rows_per_dst=[4790, 5420, 5382, 5420, 4936, 5678, 5778, 6070] combine_rows_per_src=[10480, 10928, 10904, 11068, 10646, 11056, 11194, 11322] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87578 payload_rows_remote=76652 dup_rows=43494 dup_rows_remote=38156 dedup_ratio=0.331833 router_tokens_per_expert=[472, 531, 508, 509, 537, 512, 501, 500, 497, 485, 539, 567, 513, 532, 492, 509, 514, 479, 550, 474, 479, 491, 534, 516, 504, 556, 534, 488, 519, 470, 498, 526, 542, 543, 463, 487, 456, 509, 483, 488, 520, 484, 502, 492, 493, 480, 543, 519, 522, 541, 540, 500, 529, 530, 520, 481, 515, 516, 527, 531, 526, 522, 486, 502, 523, 537, 531, 527, 573, 493, 537, 509, 480, 484, 520, 524, 522, 501, 524, 507, 530, 562, 514, 537, 510, 491, 508, 538, 531, 495, 506, 512, 521, 538, 520, 519, 534, 495, 464, 511, 510, 546, 520, 520, 536, 523, 512, 526, 489, 519, 529, 496, 483, 481, 488, 493, 484, 517, 510, 502, 551, 509, 493, 480, 504, 529, 502, 528] payload_rows_per_dst=[10870, 10926, 10804, 11036, 11000, 11080, 10966, 10896] dup_rows_per_dst=[5538, 5338, 5204, 5540, 5584, 5584, 5494, 5212] combine_rows_per_src=[10870, 10926, 10804, 11036, 11000, 11080, 10966, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87188 payload_rows_remote=76376 dup_rows=43884 dup_rows_remote=38554 dedup_ratio=0.334808 router_tokens_per_expert=[522, 470, 515, 525, 493, 543, 516, 515, 506, 483, 558, 540, 536, 536, 477, 554, 561, 497, 492, 511, 509, 454, 540, 517, 508, 495, 510, 498, 553, 494, 502, 544, 557, 514, 530, 501, 507, 534, 528, 489, 515, 525, 512, 517, 539, 544, 511, 512, 477, 496, 527, 497, 527, 517, 516, 504, 515, 546, 508, 508, 483, 492, 510, 539, 535, 526, 504, 554, 529, 508, 510, 494, 481, 496, 533, 511, 477, 521, 492, 547, 485, 492, 530, 531, 529, 526, 482, 505, 473, 534, 508, 524, 509, 507, 517, 526, 486, 488, 516, 481, 483, 528, 492, 512, 515, 542, 476, 487, 517, 511, 503, 534, 470, 524, 503, 504, 487, 507, 500, 498, 511, 469, 519, 522, 490, 525, 538, 531] payload_rows_per_dst=[10992, 10898, 11048, 10832, 10984, 10850, 10812, 10772] dup_rows_per_dst=[5586, 5472, 5622, 5492, 5452, 5506, 5330, 5424] combine_rows_per_src=[10992, 10898, 11048, 10832, 10984, 10850, 10812, 10772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627032064 dispatch_meta_bytes=458144 combine_payload_bytes=627032064 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627490208 ingress_bytes=702529536 link_bytes=702529536 token_bytes=1254522272 weight_movement_bytes=75497472 total_nvlink_bytes=1330019744 weight_to_token_ratio=0.060180 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909063402e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76518 dup_rows=43738 dup_rows_remote=38252 dedup_ratio=0.333694 router_tokens_per_expert=[566, 517, 522, 492, 517, 557, 501, 534, 522, 517, 500, 514, 503, 472, 472, 524, 527, 505, 540, 535, 505, 512, 567, 489, 540, 513, 502, 474, 543, 503, 487, 535, 547, 521, 540, 501, 497, 508, 489, 491, 516, 518, 505, 523, 475, 484, 529, 507, 503, 513, 520, 521, 557, 495, 552, 446, 494, 551, 472, 505, 513, 506, 498, 577, 544, 518, 496, 499, 491, 501, 485, 507, 504, 511, 513, 504, 495, 520, 527, 533, 518, 486, 495, 511, 499, 508, 524, 502, 517, 519, 531, 535, 522, 533, 526, 504, 515, 552, 547, 501, 531, 503, 525, 490, 511, 505, 488, 499, 551, 524, 501, 505, 483, 520, 517, 510, 499, 478, 529, 487, 508, 500, 528, 504, 504, 467, 497, 498] payload_rows_per_dst=[10946, 10962, 10816, 10938, 10860, 10948, 11068, 10796] dup_rows_per_dst=[5514, 5592, 5486, 5508, 5436, 5512, 5428, 5262] combine_rows_per_src=[10946, 10962, 10816, 10938, 10860, 10948, 11068, 10796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76476 dup_rows=43492 dup_rows_remote=37790 dedup_ratio=0.331818 router_tokens_per_expert=[476, 485, 506, 509, 521, 526, 575, 495, 482, 511, 513, 546, 515, 548, 507, 510, 535, 516, 497, 507, 492, 522, 493, 534, 541, 468, 465, 501, 478, 516, 495, 517, 501, 489, 477, 509, 510, 515, 489, 531, 520, 500, 553, 521, 492, 515, 499, 524, 538, 489, 469, 518, 492, 509, 495, 507, 491, 485, 486, 490, 532, 490, 538, 498, 532, 504, 535, 507, 504, 506, 516, 532, 520, 524, 550, 493, 507, 536, 538, 502, 562, 490, 547, 512, 520, 528, 529, 529, 518, 506, 545, 506, 534, 538, 487, 552, 505, 521, 537, 533, 501, 500, 506, 473, 556, 514, 507, 496, 496, 490, 514, 526, 496, 532, 554, 524, 512, 463, 492, 473, 533, 541, 495, 475, 540, 518, 485, 545] payload_rows_per_dst=[11012, 10894, 10942, 10726, 10994, 11104, 10834, 11074] dup_rows_per_dst=[5438, 5260, 5348, 5328, 5618, 5702, 5516, 5282] combine_rows_per_src=[11012, 10894, 10942, 10726, 10994, 11104, 10834, 11074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87314 payload_rows_remote=76338 dup_rows=43758 dup_rows_remote=38162 dedup_ratio=0.333847 router_tokens_per_expert=[487, 482, 475, 539, 494, 521, 504, 512, 577, 515, 534, 518, 536, 537, 507, 538, 491, 540, 526, 477, 523, 508, 511, 458, 492, 572, 494, 519, 485, 514, 501, 489, 480, 544, 487, 547, 506, 502, 513, 525, 492, 547, 549, 486, 524, 515, 479, 495, 487, 537, 531, 481, 526, 536, 526, 506, 534, 516, 489, 558, 524, 511, 511, 522, 541, 493, 551, 483, 522, 498, 523, 496, 548, 536, 497, 552, 530, 529, 508, 479, 499, 476, 517, 486, 514, 493, 499, 494, 519, 486, 518, 486, 505, 514, 497, 518, 510, 490, 493, 546, 513, 510, 502, 531, 493, 530, 507, 479, 530, 488, 542, 532, 520, 510, 481, 506, 539, 513, 513, 499, 499, 515, 530, 504, 541, 488, 509, 504] payload_rows_per_dst=[10920, 10818, 10930, 10978, 10976, 10820, 10858, 11014] dup_rows_per_dst=[5632, 5382, 5452, 5612, 5596, 5222, 5534, 5328] combine_rows_per_src=[10920, 10818, 10930, 10978, 10976, 10820, 10858, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87320 payload_rows_remote=76544 dup_rows=43752 dup_rows_remote=38298 dedup_ratio=0.333801 router_tokens_per_expert=[507, 555, 495, 509, 486, 551, 539, 503, 514, 504, 512, 509, 505, 499, 544, 515, 513, 517, 480, 500, 542, 481, 517, 534, 549, 534, 488, 509, 529, 534, 496, 507, 520, 528, 539, 516, 533, 505, 529, 477, 498, 477, 506, 531, 535, 483, 478, 565, 485, 495, 520, 520, 506, 485, 505, 530, 486, 498, 522, 506, 516, 509, 512, 520, 511, 547, 508, 526, 508, 532, 561, 461, 464, 495, 543, 540, 511, 504, 519, 501, 488, 500, 525, 556, 504, 518, 521, 517, 539, 488, 529, 471, 547, 502, 526, 492, 545, 478, 496, 529, 491, 492, 489, 489, 493, 503, 504, 517, 524, 511, 522, 475, 520, 479, 522, 530, 528, 531, 517, 509, 490, 534, 493, 504, 478, 546, 519, 512] payload_rows_per_dst=[11042, 10948, 11012, 10776, 10902, 10912, 10776, 10952] dup_rows_per_dst=[5452, 5512, 5428, 5454, 5560, 5534, 5340, 5472] combine_rows_per_src=[11042, 10948, 11012, 10776, 10902, 10912, 10776, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=624852992 dispatch_meta_bytes=454720 combine_payload_bytes=624852992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625307712 ingress_bytes=700350464 link_bytes=700350464 token_bytes=1250160704 weight_movement_bytes=75497472 total_nvlink_bytes=1325658176 weight_to_token_ratio=0.060390 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.881429701e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627933184 dispatch_meta_bytes=459232 combine_payload_bytes=627933184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628392416 ingress_bytes=703430656 link_bytes=703430656 token_bytes=1256325600 weight_movement_bytes=75497472 total_nvlink_bytes=1331823072 weight_to_token_ratio=0.060094 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920490872e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625672192 dispatch_meta_bytes=459720 combine_payload_bytes=625672192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626131912 ingress_bytes=701169664 link_bytes=701169664 token_bytes=1251804104 weight_movement_bytes=75497472 total_nvlink_bytes=1327301576 weight_to_token_ratio=0.060311 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.891818310e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626835456 dispatch_meta_bytes=459080 combine_payload_bytes=626835456 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627294536 ingress_bytes=702332928 link_bytes=702332928 token_bytes=1254129992 weight_movement_bytes=75497472 total_nvlink_bytes=1329627464 weight_to_token_ratio=0.060199 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906570135e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626491392 dispatch_meta_bytes=457064 combine_payload_bytes=626491392 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626948456 ingress_bytes=852983808 link_bytes=852983808 token_bytes=1253439848 weight_movement_bytes=226492416 total_nvlink_bytes=1479932264 weight_to_token_ratio=0.180697 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081703535e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625360896 dispatch_meta_bytes=458000 combine_payload_bytes=625360896 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625818896 ingress_bytes=851853312 link_bytes=851853312 token_bytes=1251179792 weight_movement_bytes=226492416 total_nvlink_bytes=1477672208 weight_to_token_ratio=0.181023 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080269907e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627048448 dispatch_meta_bytes=459368 combine_payload_bytes=627048448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627507816 ingress_bytes=702545920 link_bytes=702545920 token_bytes=1254556264 weight_movement_bytes=75497472 total_nvlink_bytes=1330053736 weight_to_token_ratio=0.060179 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909271174e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87314 payload_rows_remote=76172 dup_rows=43758 dup_rows_remote=38040 dedup_ratio=0.333847 router_tokens_per_expert=[519, 551, 568, 509, 495, 491, 518, 541, 545, 537, 518, 537, 511, 510, 522, 558, 523, 523, 469, 525, 470, 504, 480, 517, 504, 499, 527, 484, 517, 524, 507, 528, 518, 485, 529, 496, 480, 523, 499, 494, 486, 522, 495, 470, 537, 507, 564, 500, 504, 491, 515, 508, 514, 503, 517, 507, 538, 519, 480, 517, 491, 509, 538, 494, 492, 540, 488, 528, 517, 539, 541, 515, 519, 537, 525, 513, 463, 531, 542, 511, 509, 511, 510, 521, 545, 507, 558, 502, 519, 506, 518, 516, 496, 575, 486, 505, 505, 497, 476, 490, 505, 518, 495, 517, 477, 545, 502, 513, 498, 520, 490, 524, 526, 519, 481, 537, 482, 537, 511, 500, 483, 511, 450, 507, 528, 506, 533, 487] payload_rows_per_dst=[11142, 10980, 10936, 10902, 11060, 10642, 10854, 10798] dup_rows_per_dst=[5718, 5742, 5758, 5462, 5468, 4922, 5290, 5398] combine_rows_per_src=[11142, 10980, 10936, 10902, 11060, 10642, 10854, 10798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76526 dup_rows=43454 dup_rows_remote=37988 dedup_ratio=0.331528 router_tokens_per_expert=[479, 567, 508, 500, 464, 466, 553, 536, 517, 516, 478, 500, 486, 556, 515, 521, 525, 475, 497, 523, 564, 492, 524, 525, 508, 483, 477, 525, 531, 530, 550, 505, 467, 478, 498, 483, 460, 516, 509, 518, 527, 514, 499, 533, 472, 539, 545, 491, 510, 506, 540, 539, 499, 514, 513, 517, 475, 526, 500, 480, 531, 510, 500, 514, 502, 501, 554, 502, 509, 508, 501, 498, 517, 490, 522, 496, 485, 490, 518, 512, 540, 503, 510, 552, 525, 518, 509, 492, 548, 513, 540, 486, 505, 492, 516, 529, 572, 537, 486, 524, 515, 517, 496, 509, 465, 528, 535, 503, 525, 525, 530, 513, 546, 519, 507, 492, 533, 505, 546, 538, 529, 521, 517, 471, 485, 546, 496, 503] payload_rows_per_dst=[10664, 10862, 11082, 10844, 11010, 11034, 11030, 11092] dup_rows_per_dst=[5016, 5198, 5660, 5318, 5608, 5532, 5656, 5466] combine_rows_per_src=[10664, 10862, 11082, 10844, 11010, 11034, 11030, 11092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87458 payload_rows_remote=76526 dup_rows=43614 dup_rows_remote=37960 dedup_ratio=0.332748 router_tokens_per_expert=[516, 500, 503, 509, 565, 537, 506, 486, 536, 525, 501, 503, 519, 514, 511, 516, 558, 517, 497, 507, 503, 518, 518, 474, 527, 484, 508, 532, 560, 528, 517, 545, 516, 481, 502, 462, 487, 501, 544, 494, 515, 482, 541, 521, 492, 502, 480, 498, 529, 549, 531, 521, 495, 553, 508, 503, 489, 475, 474, 479, 546, 488, 533, 529, 497, 485, 475, 546, 525, 533, 516, 488, 504, 517, 478, 502, 502, 514, 517, 551, 534, 540, 489, 511, 504, 507, 514, 478, 489, 522, 513, 531, 553, 534, 527, 510, 507, 530, 518, 487, 530, 494, 512, 498, 478, 542, 514, 518, 538, 519, 509, 534, 510, 516, 470, 491, 521, 499, 536, 512, 533, 484, 537, 491, 477, 502, 534, 529] payload_rows_per_dst=[10932, 10932, 10834, 10982, 10864, 11024, 11034, 10856] dup_rows_per_dst=[5562, 5654, 5202, 5422, 5436, 5488, 5422, 5428] combine_rows_per_src=[10932, 10932, 10834, 10982, 10864, 11024, 11034, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=624001024 dispatch_meta_bytes=456848 combine_payload_bytes=624001024 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=624457872 ingress_bytes=850493440 link_bytes=850493440 token_bytes=1248458896 weight_movement_bytes=226492416 total_nvlink_bytes=1474951312 weight_to_token_ratio=0.181418 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078545398e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87192 payload_rows_remote=76194 dup_rows=43880 dup_rows_remote=38440 dedup_ratio=0.334778 router_tokens_per_expert=[494, 525, 499, 503, 540, 499, 521, 570, 499, 491, 528, 504, 486, 506, 527, 485, 537, 547, 470, 519, 492, 536, 506, 528, 522, 491, 513, 500, 523, 529, 514, 556, 496, 512, 511, 523, 490, 523, 485, 529, 558, 502, 504, 521, 509, 546, 566, 492, 513, 485, 526, 502, 485, 482, 521, 526, 561, 492, 499, 481, 512, 486, 514, 497, 510, 506, 490, 519, 501, 460, 518, 524, 497, 548, 513, 494, 540, 555, 548, 460, 536, 498, 489, 512, 517, 488, 498, 541, 538, 532, 501, 532, 529, 491, 537, 521, 495, 477, 541, 501, 563, 487, 485, 498, 508, 509, 535, 514, 531, 533, 525, 517, 484, 501, 474, 486, 504, 523, 518, 511, 490, 507, 483, 500, 520, 497, 530, 537] payload_rows_per_dst=[10778, 11006, 11024, 10828, 10826, 10978, 10998, 10754] dup_rows_per_dst=[5576, 5560, 5510, 5336, 5540, 5542, 5440, 5376] combine_rows_per_src=[10778, 11006, 11024, 10828, 10826, 10978, 10998, 10754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87174 payload_rows_remote=76276 dup_rows=43898 dup_rows_remote=38452 dedup_ratio=0.334915 router_tokens_per_expert=[526, 534, 526, 516, 500, 523, 542, 533, 494, 521, 501, 537, 509, 501, 494, 521, 517, 494, 469, 539, 517, 515, 487, 508, 521, 517, 491, 513, 475, 513, 521, 517, 517, 507, 504, 480, 499, 554, 505, 529, 533, 525, 512, 494, 490, 523, 512, 488, 550, 542, 544, 481, 524, 536, 542, 501, 513, 506, 525, 530, 508, 504, 525, 493, 463, 511, 489, 540, 543, 470, 563, 517, 518, 539, 533, 480, 495, 523, 516, 525, 504, 516, 537, 536, 513, 512, 452, 494, 506, 482, 487, 480, 547, 488, 531, 496, 519, 495, 504, 507, 460, 474, 514, 494, 568, 556, 515, 499, 501, 485, 525, 485, 482, 550, 525, 490, 506, 495, 532, 458, 558, 499, 484, 537, 546, 557, 508, 514] payload_rows_per_dst=[10890, 10858, 10898, 10944, 10956, 10834, 10818, 10976] dup_rows_per_dst=[5666, 5370, 5446, 5704, 5494, 5328, 5384, 5506] combine_rows_per_src=[10890, 10858, 10898, 10944, 10956, 10834, 10818, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76676 dup_rows=43570 dup_rows_remote=38190 dedup_ratio=0.332413 router_tokens_per_expert=[514, 486, 524, 555, 516, 527, 549, 487, 520, 484, 524, 495, 467, 532, 492, 514, 547, 512, 530, 478, 504, 533, 493, 498, 536, 501, 511, 523, 510, 517, 500, 474, 531, 483, 460, 486, 508, 510, 541, 536, 540, 490, 506, 506, 506, 502, 496, 503, 505, 535, 530, 538, 518, 491, 544, 495, 503, 512, 497, 529, 493, 538, 509, 529, 522, 542, 508, 525, 517, 484, 511, 519, 499, 537, 565, 489, 476, 556, 481, 526, 472, 477, 518, 515, 494, 513, 482, 526, 501, 509, 494, 516, 487, 560, 531, 508, 542, 517, 473, 489, 509, 496, 512, 504, 494, 496, 514, 528, 506, 489, 508, 499, 493, 502, 533, 541, 525, 518, 521, 563, 484, 527, 522, 518, 544, 510, 527, 549] payload_rows_per_dst=[10922, 10954, 10936, 10966, 11030, 10826, 10884, 10984] dup_rows_per_dst=[5450, 5380, 5272, 5566, 5484, 5380, 5268, 5770] combine_rows_per_src=[10922, 10954, 10936, 10966, 11030, 10826, 10884, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87930 payload_rows_remote=77050 dup_rows=43142 dup_rows_remote=37828 dedup_ratio=0.329147 router_tokens_per_expert=[552, 484, 520, 513, 508, 568, 502, 515, 491, 523, 518, 530, 534, 526, 515, 522, 520, 546, 496, 488, 491, 491, 501, 516, 483, 503, 504, 550, 523, 491, 521, 479, 532, 488, 557, 481, 521, 510, 502, 534, 486, 487, 520, 525, 525, 489, 514, 511, 478, 516, 503, 527, 502, 501, 507, 525, 531, 534, 469, 488, 520, 488, 508, 500, 494, 492, 530, 538, 498, 538, 502, 526, 526, 519, 540, 482, 521, 473, 496, 537, 511, 537, 500, 525, 544, 503, 552, 522, 546, 497, 476, 486, 489, 513, 522, 525, 498, 514, 481, 490, 486, 554, 491, 519, 503, 479, 504, 527, 508, 516, 543, 531, 514, 569, 505, 509, 514, 486, 511, 492, 497, 516, 510, 517, 525, 520, 536, 508] payload_rows_per_dst=[11202, 10918, 10950, 10880, 10982, 10974, 10980, 11044] dup_rows_per_dst=[5440, 5288, 5414, 5314, 5442, 5522, 5308, 5414] combine_rows_per_src=[11202, 10918, 10950, 10880, 10982, 10974, 10980, 11044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87516 payload_rows_remote=76762 dup_rows=43556 dup_rows_remote=38318 dedup_ratio=0.332306 router_tokens_per_expert=[539, 534, 550, 510, 497, 519, 491, 509, 521, 499, 500, 543, 522, 537, 510, 497, 509, 540, 509, 518, 513, 487, 493, 524, 519, 512, 509, 507, 510, 528, 516, 547, 517, 520, 525, 531, 522, 503, 535, 497, 495, 548, 490, 505, 525, 507, 540, 495, 493, 501, 495, 513, 519, 520, 498, 537, 502, 520, 508, 568, 529, 505, 491, 495, 492, 494, 509, 497, 498, 503, 501, 528, 511, 469, 512, 476, 495, 512, 519, 480, 478, 559, 516, 508, 523, 517, 484, 517, 483, 540, 524, 515, 507, 513, 471, 528, 520, 531, 505, 531, 525, 546, 461, 492, 528, 541, 488, 501, 519, 519, 541, 473, 492, 500, 534, 479, 500, 480, 493, 531, 518, 529, 504, 514, 511, 516, 533, 534] payload_rows_per_dst=[11008, 10938, 11008, 11012, 10754, 10938, 10944, 10914] dup_rows_per_dst=[5548, 5544, 5502, 5376, 5238, 5428, 5498, 5422] combine_rows_per_src=[11008, 10938, 11008, 11012, 10754, 10938, 10944, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626900992 dispatch_meta_bytes=458056 combine_payload_bytes=626900992 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627359048 ingress_bytes=777895936 link_bytes=777895936 token_bytes=1254260040 weight_movement_bytes=150994944 total_nvlink_bytes=1405254984 weight_to_token_ratio=0.120386 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864815440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626900992 dispatch_meta_bytes=457944 combine_payload_bytes=626900992 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627358936 ingress_bytes=777895936 link_bytes=777895936 token_bytes=1254259928 weight_movement_bytes=150994944 total_nvlink_bytes=1405254872 weight_to_token_ratio=0.120386 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864815440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=624181248 dispatch_meta_bytes=458536 combine_payload_bytes=624181248 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624639784 ingress_bytes=699678720 link_bytes=699678720 token_bytes=1248821032 weight_movement_bytes=75497472 total_nvlink_bytes=1324318504 weight_to_token_ratio=0.060455 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.872911042e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=624852992 dispatch_meta_bytes=458912 combine_payload_bytes=624852992 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625311904 ingress_bytes=700350464 link_bytes=700350464 token_bytes=1250164896 weight_movement_bytes=75497472 total_nvlink_bytes=1325662368 weight_to_token_ratio=0.060390 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.881429701e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=459464 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628589256 ingress_bytes=779124736 link_bytes=779124736 token_bytes=1256719048 weight_movement_bytes=150994944 total_nvlink_bytes=1407713992 weight_to_token_ratio=0.120150 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.880398353e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=631193600 dispatch_meta_bytes=459512 combine_payload_bytes=631193600 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=631653112 ingress_bytes=857686016 link_bytes=857686016 token_bytes=1262846712 weight_movement_bytes=226492416 total_nvlink_bytes=1489339128 weight_to_token_ratio=0.179351 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.087666597e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628834304 dispatch_meta_bytes=460320 combine_payload_bytes=628834304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629294624 ingress_bytes=704331776 link_bytes=704331776 token_bytes=1258128928 weight_movement_bytes=75497472 total_nvlink_bytes=1333626400 weight_to_token_ratio=0.060008 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.931918342e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15038889560 prefetch_bytes=830472192 reduce_bytes=528482304 weight_movement_bytes=1358954496 egress_bytes=7522188888 ingress_bytes=8875655168 link_bytes=8875655168 total_nvlink_bytes=16397844056 weight_to_token_ratio=0.090363 prefetch_share_of_ingress=0.093567 dedup_ratio=0.332142 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=7 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.379650080e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15035795584 prefetch_bytes=905969664 reduce_bytes=679477248 weight_movement_bytes=1585446912 egress_bytes=7520651392 ingress_bytes=9100591104 link_bytes=9100591104 total_nvlink_bytes=16621242496 weight_to_token_ratio=0.105445 prefetch_share_of_ingress=0.099551 dedup_ratio=0.333388 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=9 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.617358771e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87628 payload_rows_remote=76638 dup_rows=43444 dup_rows_remote=38016 dedup_ratio=0.331451 router_tokens_per_expert=[491, 511, 555, 489, 506, 522, 506, 509, 532, 482, 475, 474, 489, 512, 515, 514, 543, 537, 516, 477, 505, 510, 496, 517, 529, 553, 522, 537, 517, 501, 517, 505, 525, 508, 517, 501, 504, 504, 510, 480, 490, 532, 547, 535, 514, 501, 480, 519, 523, 497, 538, 543, 527, 495, 489, 509, 537, 496, 527, 527, 523, 504, 500, 555, 531, 526, 504, 491, 494, 528, 503, 498, 495, 500, 562, 522, 483, 529, 524, 514, 503, 517, 515, 527, 492, 492, 495, 518, 519, 497, 499, 495, 527, 516, 521, 512, 530, 537, 536, 516, 503, 525, 478, 488, 485, 504, 538, 535, 519, 493, 523, 494, 545, 480, 460, 486, 496, 530, 521, 534, 526, 536, 509, 456, 495, 542, 540, 506] payload_rows_per_dst=[10990, 11014, 10898, 11042, 10888, 10954, 11020, 10822] dup_rows_per_dst=[5428, 5596, 5436, 5538, 5520, 5290, 5388, 5248] combine_rows_per_src=[10990, 11014, 10898, 11042, 10888, 10954, 11020, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15052931176 prefetch_bytes=905969664 reduce_bytes=301989888 weight_movement_bytes=1207959552 egress_bytes=7529218152 ingress_bytes=8731672576 link_bytes=8731672576 total_nvlink_bytes=16260890728 weight_to_token_ratio=0.080247 prefetch_share_of_ingress=0.103757 dedup_ratio=0.332537 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=4 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.227491585e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15037990016 prefetch_bytes=905969664 reduce_bytes=679477248 weight_movement_bytes=1585446912 egress_bytes=7521748096 ingress_bytes=9101688832 link_bytes=9101688832 total_nvlink_bytes=16623436928 weight_to_token_ratio=0.105429 prefetch_share_of_ingress=0.099539 dedup_ratio=0.333430 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=9 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.618518832e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15053647928 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=7529574456 ingress_bytes=9185017856 link_bytes=9185017856 total_nvlink_bytes=16714592312 weight_to_token_ratio=0.110335 prefetch_share_of_ingress=0.090416 dedup_ratio=0.332335 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041667 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.706579609e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15057679656 prefetch_bytes=679477248 reduce_bytes=981467136 weight_movement_bytes=1660944384 egress_bytes=7531590952 ingress_bytes=9187033088 link_bytes=9187033088 total_nvlink_bytes=16718624040 weight_to_token_ratio=0.110305 prefetch_share_of_ingress=0.073960 dedup_ratio=0.332450 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=13 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.708709274e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15058697560 prefetch_bytes=528482304 reduce_bytes=1660944384 weight_movement_bytes=2189426688 egress_bytes=7532100952 ingress_bytes=9716023296 link_bytes=9716023296 total_nvlink_bytes=17248124248 weight_to_token_ratio=0.145393 prefetch_share_of_ingress=0.054393 dedup_ratio=0.332523 prefetch_slots=14 unused_prefetch_slots=370 slot_utilization=0.036458 reduce_slots=22 remote_demand=7 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=26797 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.026773764e-04 prefetch_bytes_per_gemm_flop=5.584916258e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=3 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627818496 dispatch_meta_bytes=458616 combine_payload_bytes=627818496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628277112 ingress_bytes=778813440 link_bytes=778813440 token_bytes=1256095608 weight_movement_bytes=150994944 total_nvlink_bytes=1407090552 weight_to_token_ratio=0.120210 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876450682e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=3 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15021064096 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=7513276320 ingress_bytes=8942239744 link_bytes=8942239744 total_nvlink_bytes=16455516064 weight_to_token_ratio=0.095496 prefetch_share_of_ingress=0.084428 dedup_ratio=0.332868 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=9 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.450015592e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none + [2026-08-29 23:36:23] iteration 4/ 12 | consumed samples: 32 | elapsed time per iteration (ms): 354.7 | throughput per GPU (TFLOP/s/GPU): 509.8 | learning rate: 8.000000E-05 | global batch size: 8 | lm loss: 1.340185E+01 | loss scale: 1.0 | grad norm: 19.519 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87402 payload_rows_remote=76562 dup_rows=43670 dup_rows_remote=38460 dedup_ratio=0.333176 router_tokens_per_expert=[526, 503, 512, 484, 531, 497, 508, 521, 502, 481, 539, 489, 557, 483, 521, 499, 493, 546, 485, 488, 474, 500, 517, 495, 537, 477, 476, 521, 490, 512, 497, 517, 510, 491, 518, 555, 530, 528, 509, 518, 513, 534, 503, 523, 499, 482, 481, 477, 479, 557, 515, 558, 515, 522, 521, 533, 525, 525, 509, 513, 514, 498, 484, 512, 518, 511, 532, 564, 500, 518, 537, 523, 510, 488, 490, 503, 496, 545, 478, 503, 480, 479, 544, 504, 558, 518, 512, 553, 477, 525, 553, 496, 497, 510, 531, 502, 534, 514, 505, 510, 508, 489, 510, 543, 551, 521, 480, 541, 513, 520, 518, 505, 543, 495, 502, 499, 511, 540, 511, 487, 481, 504, 540, 540, 516, 530, 505, 486] payload_rows_per_dst=[10934, 10840, 10974, 11000, 10894, 10940, 10916, 10904] dup_rows_per_dst=[5372, 5210, 5368, 5560, 5538, 5538, 5608, 5476] combine_rows_per_src=[10934, 10840, 10974, 11000, 10894, 10940, 10916, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87462 payload_rows_remote=76558 dup_rows=43610 dup_rows_remote=38012 dedup_ratio=0.332718 router_tokens_per_expert=[515, 547, 503, 528, 481, 511, 519, 536, 522, 510, 492, 530, 538, 464, 523, 515, 529, 489, 502, 513, 529, 487, 529, 508, 513, 535, 501, 533, 525, 519, 502, 493, 509, 501, 537, 509, 500, 541, 514, 514, 510, 488, 519, 544, 542, 537, 520, 473, 548, 485, 526, 547, 517, 510, 540, 511, 518, 493, 480, 449, 517, 535, 510, 503, 506, 536, 531, 527, 510, 541, 488, 500, 480, 497, 500, 526, 503, 521, 512, 504, 507, 505, 500, 548, 522, 498, 484, 542, 536, 484, 521, 511, 488, 508, 497, 489, 536, 494, 480, 531, 492, 528, 515, 510, 486, 494, 533, 503, 503, 500, 511, 528, 514, 508, 526, 490, 503, 507, 552, 523, 495, 533, 529, 518, 525, 483, 510, 466] payload_rows_per_dst=[10976, 10972, 10880, 10982, 10904, 10888, 10956, 10904] dup_rows_per_dst=[5492, 5442, 5472, 5396, 5216, 5418, 5576, 5598] combine_rows_per_src=[10976, 10972, 10880, 10982, 10904, 10888, 10956, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87296 payload_rows_remote=76352 dup_rows=43776 dup_rows_remote=38364 dedup_ratio=0.333984 router_tokens_per_expert=[488, 506, 510, 500, 533, 487, 532, 514, 526, 471, 519, 539, 535, 484, 467, 492, 505, 508, 497, 482, 527, 495, 505, 508, 555, 507, 519, 520, 567, 494, 516, 508, 512, 491, 482, 484, 504, 542, 538, 514, 494, 501, 505, 527, 527, 532, 506, 519, 497, 479, 547, 549, 509, 482, 529, 461, 474, 493, 500, 503, 512, 531, 503, 503, 492, 500, 510, 533, 506, 542, 493, 458, 505, 499, 547, 498, 522, 514, 510, 556, 482, 552, 524, 507, 491, 553, 491, 483, 546, 487, 511, 513, 540, 538, 572, 522, 531, 530, 477, 538, 525, 493, 516, 529, 540, 525, 531, 481, 561, 537, 495, 517, 499, 503, 500, 489, 518, 490, 557, 549, 521, 506, 489, 471, 520, 510, 512, 513] payload_rows_per_dst=[10866, 10934, 10944, 10786, 10848, 11062, 10928, 10928] dup_rows_per_dst=[5340, 5492, 5412, 5358, 5522, 5562, 5724, 5366] combine_rows_per_src=[10866, 10934, 10944, 10786, 10848, 11062, 10928, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87246 payload_rows_remote=76340 dup_rows=43826 dup_rows_remote=38296 dedup_ratio=0.334366 router_tokens_per_expert=[498, 499, 520, 523, 511, 540, 490, 504, 493, 529, 525, 509, 493, 531, 502, 506, 553, 564, 528, 498, 496, 493, 527, 475, 520, 540, 489, 473, 545, 511, 465, 485, 524, 525, 507, 532, 543, 493, 467, 510, 488, 493, 468, 529, 494, 513, 530, 510, 507, 517, 498, 483, 540, 538, 484, 486, 529, 510, 506, 519, 530, 496, 486, 536, 504, 511, 518, 526, 526, 507, 485, 512, 479, 552, 506, 516, 491, 522, 512, 504, 525, 522, 514, 499, 544, 523, 512, 501, 516, 489, 496, 505, 561, 499, 489, 523, 517, 533, 526, 551, 524, 518, 509, 508, 528, 478, 552, 536, 485, 544, 514, 508, 507, 504, 525, 525, 516, 499, 525, 547, 518, 513, 482, 530, 520, 498, 479, 502] payload_rows_per_dst=[10900, 10786, 10818, 10920, 10946, 10906, 11068, 10902] dup_rows_per_dst=[5446, 5538, 5434, 5410, 5396, 5530, 5594, 5478] combine_rows_per_src=[10900, 10786, 10818, 10920, 10946, 10906, 11068, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87404 payload_rows_remote=76532 dup_rows=43668 dup_rows_remote=38176 dedup_ratio=0.333160 router_tokens_per_expert=[540, 497, 513, 512, 475, 504, 511, 522, 519, 533, 497, 514, 476, 501, 521, 524, 525, 511, 545, 526, 520, 513, 549, 521, 494, 517, 510, 510, 537, 529, 505, 544, 478, 543, 522, 516, 545, 523, 527, 526, 486, 515, 489, 518, 492, 498, 462, 528, 452, 506, 527, 528, 531, 489, 507, 501, 497, 524, 537, 487, 534, 516, 455, 502, 498, 525, 514, 525, 497, 489, 532, 525, 520, 497, 533, 545, 522, 523, 493, 539, 490, 497, 484, 537, 521, 524, 517, 497, 521, 527, 485, 527, 493, 473, 504, 475, 538, 492, 523, 496, 514, 524, 520, 500, 533, 493, 514, 479, 502, 520, 546, 488, 540, 495, 555, 507, 464, 510, 495, 532, 517, 502, 517, 518, 514, 519, 497, 547] payload_rows_per_dst=[10872, 11046, 10984, 10790, 11036, 10854, 10872, 10950] dup_rows_per_dst=[5446, 5666, 5352, 5396, 5518, 5290, 5492, 5508] combine_rows_per_src=[10872, 11046, 10984, 10790, 11036, 10854, 10872, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=460088 combine_payload_bytes=627195904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627655992 ingress_bytes=702693376 link_bytes=702693376 token_bytes=1254851896 weight_movement_bytes=75497472 total_nvlink_bytes=1330349368 weight_to_token_ratio=0.060164 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911141123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627163136 dispatch_meta_bytes=458280 combine_payload_bytes=627163136 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627621416 ingress_bytes=702660608 link_bytes=702660608 token_bytes=1254784552 weight_movement_bytes=75497472 total_nvlink_bytes=1330282024 weight_to_token_ratio=0.060168 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910725579e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87594 payload_rows_remote=76656 dup_rows=43478 dup_rows_remote=38028 dedup_ratio=0.331711 router_tokens_per_expert=[478, 547, 500, 506, 512, 505, 527, 531, 507, 478, 498, 534, 532, 510, 508, 522, 496, 480, 549, 490, 537, 516, 493, 533, 486, 533, 509, 554, 553, 471, 543, 509, 507, 487, 514, 515, 550, 516, 516, 495, 482, 510, 545, 503, 551, 508, 509, 519, 508, 515, 511, 480, 501, 511, 502, 505, 532, 506, 512, 527, 537, 501, 510, 536, 518, 480, 541, 507, 489, 518, 522, 505, 551, 487, 493, 500, 472, 488, 488, 542, 511, 536, 489, 497, 557, 488, 503, 518, 518, 495, 561, 524, 500, 520, 509, 492, 540, 497, 506, 511, 500, 502, 543, 542, 535, 535, 538, 523, 458, 526, 521, 488, 530, 516, 529, 504, 455, 463, 510, 514, 513, 483, 520, 494, 482, 532, 514, 525] payload_rows_per_dst=[10972, 11024, 11020, 10938, 10832, 10920, 10968, 10920] dup_rows_per_dst=[5418, 5480, 5434, 5450, 5370, 5516, 5562, 5248] combine_rows_per_src=[10972, 11024, 11020, 10938, 10832, 10920, 10968, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625475584 dispatch_meta_bytes=458864 combine_payload_bytes=625475584 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=625934448 ingress_bytes=776470528 link_bytes=776470528 token_bytes=1251410032 weight_movement_bytes=150994944 total_nvlink_bytes=1402404976 weight_to_token_ratio=0.120660 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.846739260e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625377280 dispatch_meta_bytes=458544 combine_payload_bytes=625377280 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625835824 ingress_bytes=851869696 link_bytes=851869696 token_bytes=1251213104 weight_movement_bytes=226492416 total_nvlink_bytes=1477705520 weight_to_token_ratio=0.181018 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080290684e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76496 dup_rows=43554 dup_rows_remote=37714 dedup_ratio=0.332291 router_tokens_per_expert=[522, 525, 503, 513, 540, 494, 527, 511, 497, 519, 500, 535, 483, 497, 493, 478, 515, 533, 518, 550, 493, 511, 510, 503, 539, 516, 492, 518, 498, 509, 489, 497, 560, 529, 498, 501, 528, 505, 488, 503, 512, 531, 527, 519, 465, 464, 480, 538, 503, 503, 527, 484, 506, 538, 480, 524, 456, 522, 534, 476, 562, 507, 482, 515, 515, 539, 529, 509, 519, 528, 508, 540, 577, 532, 536, 505, 500, 530, 536, 528, 493, 512, 451, 545, 545, 502, 497, 561, 537, 512, 507, 492, 497, 526, 496, 483, 503, 494, 497, 553, 548, 474, 498, 480, 529, 502, 520, 516, 490, 502, 524, 505, 492, 486, 532, 520, 464, 538, 531, 517, 503, 526, 510, 523, 572, 497, 518, 490] payload_rows_per_dst=[10822, 10914, 10890, 10860, 11022, 11000, 11014, 10996] dup_rows_per_dst=[5452, 5468, 5406, 5378, 5840, 5312, 5256, 5442] combine_rows_per_src=[10822, 10914, 10890, 10860, 11022, 11000, 11014, 10996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626950144 dispatch_meta_bytes=458832 combine_payload_bytes=626950144 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627408976 ingress_bytes=777945088 link_bytes=777945088 token_bytes=1254359120 weight_movement_bytes=150994944 total_nvlink_bytes=1405354064 weight_to_token_ratio=0.120376 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.865438756e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87524 payload_rows_remote=76478 dup_rows=43548 dup_rows_remote=37830 dedup_ratio=0.332245 router_tokens_per_expert=[520, 504, 526, 483, 541, 510, 522, 515, 504, 497, 483, 511, 509, 538, 540, 529, 516, 499, 475, 485, 502, 474, 520, 477, 511, 523, 526, 535, 469, 526, 519, 539, 527, 516, 522, 531, 527, 505, 472, 560, 536, 527, 501, 510, 522, 552, 555, 479, 517, 502, 498, 551, 482, 448, 461, 524, 502, 521, 495, 517, 510, 496, 515, 482, 502, 537, 517, 468, 458, 548, 525, 538, 527, 498, 497, 558, 516, 487, 520, 517, 520, 534, 507, 552, 494, 554, 468, 522, 527, 494, 531, 506, 521, 492, 520, 492, 501, 509, 502, 525, 515, 526, 524, 488, 491, 497, 522, 530, 486, 514, 526, 516, 520, 527, 509, 475, 541, 505, 520, 511, 519, 512, 523, 535, 524, 485, 518, 502] payload_rows_per_dst=[11046, 10884, 11164, 11128, 10866, 10862, 10626, 10948] dup_rows_per_dst=[5718, 5376, 5520, 5720, 5324, 5474, 4912, 5504] combine_rows_per_src=[11046, 10884, 11164, 11128, 10866, 10862, 10626, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627965952 dispatch_meta_bytes=458736 combine_payload_bytes=627965952 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628424688 ingress_bytes=703463424 link_bytes=703463424 token_bytes=1256390640 weight_movement_bytes=75497472 total_nvlink_bytes=1331888112 weight_to_token_ratio=0.060091 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920906416e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626655232 dispatch_meta_bytes=456840 combine_payload_bytes=626655232 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627112072 ingress_bytes=853147648 link_bytes=853147648 token_bytes=1253767304 weight_movement_bytes=226492416 total_nvlink_bytes=1480259720 weight_to_token_ratio=0.180649 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081911307e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87612 payload_rows_remote=76576 dup_rows=43460 dup_rows_remote=38054 dedup_ratio=0.331573 router_tokens_per_expert=[543, 553, 512, 483, 494, 537, 511, 522, 518, 502, 530, 496, 464, 492, 553, 511, 488, 551, 472, 507, 561, 539, 488, 493, 508, 500, 529, 513, 518, 508, 529, 517, 525, 543, 485, 524, 482, 496, 500, 506, 530, 483, 531, 478, 518, 518, 501, 517, 493, 513, 491, 487, 540, 509, 506, 491, 518, 517, 514, 493, 555, 550, 482, 534, 509, 535, 486, 505, 503, 529, 528, 521, 585, 521, 514, 473, 509, 494, 498, 473, 523, 524, 494, 485, 522, 489, 550, 504, 505, 513, 535, 476, 501, 519, 533, 516, 515, 528, 511, 534, 493, 512, 541, 538, 572, 528, 495, 518, 504, 531, 507, 499, 501, 488, 505, 526, 486, 502, 465, 509, 483, 526, 511, 488, 534, 501, 554, 487] payload_rows_per_dst=[11020, 11036, 10812, 10946, 10930, 10924, 11120, 10824] dup_rows_per_dst=[5422, 5406, 5462, 5440, 5436, 5454, 5532, 5308] combine_rows_per_src=[11020, 11036, 10812, 10946, 10930, 10924, 11120, 10824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87674 payload_rows_remote=76524 dup_rows=43398 dup_rows_remote=37946 dedup_ratio=0.331100 router_tokens_per_expert=[495, 490, 528, 486, 498, 498, 505, 463, 498, 502, 552, 500, 518, 511, 528, 569, 523, 523, 498, 537, 551, 495, 530, 474, 528, 477, 518, 507, 473, 519, 471, 535, 481, 521, 550, 507, 519, 488, 544, 529, 509, 520, 529, 524, 529, 487, 499, 485, 502, 499, 521, 464, 507, 544, 535, 479, 481, 517, 508, 513, 542, 531, 501, 506, 496, 496, 488, 497, 487, 487, 536, 513, 510, 540, 531, 523, 502, 556, 523, 488, 540, 533, 496, 508, 517, 538, 476, 523, 499, 492, 529, 501, 500, 531, 496, 562, 505, 512, 540, 535, 519, 491, 485, 475, 510, 510, 527, 494, 541, 516, 531, 508, 510, 520, 529, 511, 492, 566, 482, 526, 541, 526, 482, 527, 481, 501, 544, 514] payload_rows_per_dst=[10500, 11078, 10906, 10940, 11196, 10908, 10996, 11150] dup_rows_per_dst=[4822, 5574, 5536, 5306, 5732, 5342, 5634, 5452] combine_rows_per_src=[10500, 11078, 10906, 10940, 11196, 10908, 10996, 11150] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626507776 dispatch_meta_bytes=457232 combine_payload_bytes=626507776 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626965008 ingress_bytes=702005248 link_bytes=702005248 token_bytes=1253472784 weight_movement_bytes=75497472 total_nvlink_bytes=1328970256 weight_to_token_ratio=0.060231 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.902414692e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87796 payload_rows_remote=76736 dup_rows=43276 dup_rows_remote=37762 dedup_ratio=0.330170 router_tokens_per_expert=[452, 530, 532, 495, 497, 521, 531, 575, 537, 470, 512, 525, 539, 494, 503, 516, 467, 523, 516, 506, 476, 468, 504, 573, 500, 519, 514, 506, 529, 498, 532, 506, 519, 518, 512, 508, 507, 516, 503, 531, 486, 536, 542, 497, 486, 546, 516, 537, 519, 521, 522, 531, 503, 509, 523, 511, 504, 522, 498, 513, 528, 474, 507, 469, 512, 504, 473, 521, 484, 516, 545, 506, 477, 518, 524, 522, 503, 489, 485, 544, 476, 502, 502, 519, 495, 521, 527, 524, 566, 524, 525, 512, 525, 512, 529, 528, 513, 485, 494, 511, 520, 510, 528, 526, 508, 492, 524, 540, 503, 518, 510, 473, 484, 529, 502, 498, 512, 505, 499, 513, 466, 547, 523, 509, 521, 535, 519, 529] payload_rows_per_dst=[10986, 10974, 10966, 10914, 10958, 11060, 10952, 10986] dup_rows_per_dst=[5472, 5300, 5554, 5394, 5288, 5514, 5358, 5396] combine_rows_per_src=[10986, 10974, 10966, 10914, 10958, 11060, 10952, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87170 payload_rows_remote=76336 dup_rows=43902 dup_rows_remote=38368 dedup_ratio=0.334946 router_tokens_per_expert=[499, 547, 522, 523, 534, 533, 569, 538, 530, 486, 508, 533, 500, 539, 538, 525, 504, 502, 507, 482, 510, 497, 534, 509, 521, 485, 505, 550, 504, 516, 530, 472, 540, 566, 498, 503, 493, 492, 524, 529, 519, 497, 517, 499, 489, 508, 537, 473, 533, 516, 539, 528, 525, 497, 521, 562, 549, 524, 497, 530, 494, 510, 508, 522, 510, 539, 493, 534, 452, 484, 539, 549, 520, 506, 509, 497, 496, 504, 521, 534, 514, 506, 485, 540, 520, 479, 509, 497, 537, 528, 491, 475, 505, 478, 519, 504, 485, 479, 487, 487, 496, 503, 503, 510, 515, 461, 504, 570, 492, 494, 537, 487, 541, 502, 533, 527, 534, 504, 491, 513, 450, 513, 508, 519, 480, 538, 495, 513] payload_rows_per_dst=[11168, 10720, 10834, 10986, 10912, 10866, 10764, 10920] dup_rows_per_dst=[5680, 5536, 5534, 5724, 5462, 5308, 5256, 5402] combine_rows_per_src=[11168, 10720, 10834, 10986, 10912, 10866, 10764, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87392 payload_rows_remote=76378 dup_rows=43680 dup_rows_remote=38266 dedup_ratio=0.333252 router_tokens_per_expert=[531, 544, 519, 502, 520, 528, 487, 504, 547, 497, 467, 562, 505, 502, 529, 499, 501, 502, 504, 535, 482, 480, 522, 495, 498, 559, 505, 459, 497, 558, 521, 534, 530, 517, 517, 516, 536, 530, 517, 509, 499, 504, 483, 507, 501, 507, 525, 486, 512, 549, 498, 524, 479, 499, 486, 529, 498, 491, 517, 564, 496, 491, 519, 506, 491, 513, 484, 482, 550, 538, 514, 507, 526, 501, 482, 489, 557, 512, 493, 483, 480, 522, 501, 486, 506, 520, 476, 491, 510, 564, 534, 494, 508, 524, 536, 511, 512, 505, 515, 538, 465, 531, 520, 505, 522, 514, 491, 491, 546, 531, 542, 486, 501, 522, 551, 467, 496, 510, 496, 530, 528, 566, 527, 564, 523, 503, 526, 490] payload_rows_per_dst=[11010, 10904, 10888, 10808, 10866, 10950, 11014, 10952] dup_rows_per_dst=[5476, 5400, 5480, 5508, 5378, 5376, 5414, 5648] combine_rows_per_src=[11010, 10904, 10888, 10808, 10866, 10950, 11014, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627310592 dispatch_meta_bytes=458520 combine_payload_bytes=627310592 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627769112 ingress_bytes=702808064 link_bytes=702808064 token_bytes=1255079704 weight_movement_bytes=75497472 total_nvlink_bytes=1330577176 weight_to_token_ratio=0.060154 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912595529e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=457880 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627342488 ingress_bytes=777879552 link_bytes=777879552 token_bytes=1254227096 weight_movement_bytes=150994944 total_nvlink_bytes=1405222040 weight_to_token_ratio=0.120389 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864607668e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628621312 dispatch_meta_bytes=457992 combine_payload_bytes=628621312 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=629079304 ingress_bytes=779616256 link_bytes=779616256 token_bytes=1257700616 weight_movement_bytes=150994944 total_nvlink_bytes=1408695560 weight_to_token_ratio=0.120056 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.886631519e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625344512 dispatch_meta_bytes=458816 combine_payload_bytes=625344512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625803328 ingress_bytes=700841984 link_bytes=700841984 token_bytes=1251147840 weight_movement_bytes=75497472 total_nvlink_bytes=1326645312 weight_to_token_ratio=0.060343 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.887662867e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625688576 dispatch_meta_bytes=458576 combine_payload_bytes=625688576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626147152 ingress_bytes=701186048 link_bytes=701186048 token_bytes=1251835728 weight_movement_bytes=75497472 total_nvlink_bytes=1327333200 weight_to_token_ratio=0.060309 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892026083e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87498 payload_rows_remote=76634 dup_rows=43574 dup_rows_remote=38066 dedup_ratio=0.332443 router_tokens_per_expert=[527, 502, 494, 536, 529, 498, 524, 560, 488, 502, 532, 539, 517, 498, 528, 513, 456, 494, 511, 500, 512, 548, 516, 474, 509, 497, 490, 513, 531, 530, 540, 552, 462, 510, 572, 513, 545, 488, 501, 540, 508, 509, 488, 509, 458, 499, 508, 525, 542, 537, 474, 518, 509, 534, 526, 536, 506, 495, 518, 499, 475, 518, 514, 485, 535, 515, 530, 498, 496, 468, 514, 488, 510, 512, 514, 508, 505, 536, 499, 501, 490, 509, 542, 533, 488, 500, 538, 524, 524, 522, 502, 500, 521, 489, 505, 547, 485, 528, 486, 505, 506, 518, 490, 521, 562, 524, 500, 488, 523, 504, 523, 527, 508, 499, 496, 498, 539, 476, 526, 520, 530, 526, 542, 493, 495, 521, 546, 487] payload_rows_per_dst=[11036, 10944, 10844, 10864, 10886, 10990, 10906, 11028] dup_rows_per_dst=[5538, 5402, 5426, 5508, 5372, 5478, 5474, 5376] combine_rows_per_src=[11036, 10944, 10844, 10864, 10886, 10990, 10906, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76630 dup_rows=43598 dup_rows_remote=38272 dedup_ratio=0.332626 router_tokens_per_expert=[510, 503, 513, 523, 514, 499, 551, 487, 506, 485, 529, 516, 507, 569, 509, 557, 505, 559, 522, 521, 510, 485, 519, 482, 524, 487, 490, 478, 495, 545, 504, 481, 487, 529, 537, 468, 519, 564, 505, 485, 528, 524, 493, 507, 532, 523, 524, 511, 533, 485, 532, 493, 519, 474, 522, 497, 526, 513, 520, 472, 499, 550, 504, 572, 474, 563, 475, 518, 490, 511, 535, 504, 511, 474, 527, 496, 507, 483, 494, 523, 518, 486, 569, 509, 477, 535, 518, 483, 565, 528, 476, 498, 527, 559, 507, 539, 470, 501, 501, 523, 561, 483, 499, 488, 500, 515, 496, 487, 521, 466, 506, 534, 482, 499, 495, 522, 501, 528, 533, 511, 530, 516, 509, 515, 565, 515, 538, 515] payload_rows_per_dst=[11084, 10872, 10902, 10980, 10844, 11000, 10852, 10940] dup_rows_per_dst=[5472, 5342, 5570, 5442, 5326, 5588, 5250, 5608] combine_rows_per_src=[11084, 10872, 10902, 10980, 10844, 11000, 10852, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87492 payload_rows_remote=76508 dup_rows=43580 dup_rows_remote=38178 dedup_ratio=0.332489 router_tokens_per_expert=[511, 540, 532, 515, 475, 495, 525, 541, 496, 486, 490, 511, 517, 501, 537, 517, 535, 507, 539, 466, 498, 500, 505, 533, 535, 545, 530, 551, 481, 531, 478, 558, 509, 479, 495, 542, 499, 499, 501, 554, 502, 499, 494, 492, 509, 494, 509, 514, 537, 483, 517, 523, 528, 555, 524, 498, 488, 474, 505, 510, 499, 518, 490, 507, 527, 449, 524, 516, 522, 548, 549, 519, 518, 530, 478, 479, 530, 514, 514, 526, 540, 515, 489, 518, 506, 477, 498, 497, 504, 501, 505, 501, 520, 512, 545, 533, 521, 506, 497, 529, 534, 506, 538, 496, 514, 524, 490, 533, 505, 554, 487, 541, 526, 506, 499, 483, 478, 527, 516, 525, 504, 522, 525, 560, 458, 516, 487, 497] payload_rows_per_dst=[10984, 11094, 10844, 10910, 10954, 10862, 11002, 10842] dup_rows_per_dst=[5402, 5490, 5514, 5402, 5532, 5452, 5548, 5240] combine_rows_per_src=[10984, 11094, 10844, 10910, 10954, 10862, 11002, 10842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87478 payload_rows_remote=76468 dup_rows=43594 dup_rows_remote=37964 dedup_ratio=0.332596 router_tokens_per_expert=[493, 525, 510, 496, 484, 535, 493, 505, 529, 496, 507, 525, 533, 529, 511, 515, 518, 526, 479, 510, 532, 507, 473, 497, 529, 506, 506, 509, 530, 518, 507, 542, 517, 475, 546, 486, 544, 510, 500, 504, 502, 490, 510, 519, 519, 544, 551, 509, 545, 541, 520, 523, 525, 546, 499, 536, 507, 538, 487, 535, 497, 517, 479, 522, 511, 489, 529, 503, 504, 532, 503, 485, 493, 494, 550, 513, 517, 504, 519, 533, 524, 496, 489, 522, 470, 471, 478, 512, 490, 527, 544, 550, 537, 524, 509, 516, 503, 499, 483, 462, 560, 538, 466, 508, 485, 480, 505, 483, 525, 539, 498, 469, 568, 523, 525, 499, 508, 503, 476, 551, 515, 520, 517, 520, 496, 520, 507, 529] payload_rows_per_dst=[10928, 10970, 10930, 10934, 11004, 10966, 10736, 11010] dup_rows_per_dst=[5444, 5366, 5540, 5382, 5354, 5352, 5526, 5630] combine_rows_per_src=[10928, 10970, 10930, 10934, 11004, 10966, 10736, 11010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87752 payload_rows_remote=76844 dup_rows=43320 dup_rows_remote=38110 dedup_ratio=0.330505 router_tokens_per_expert=[537, 555, 531, 461, 555, 509, 518, 502, 532, 495, 511, 466, 485, 505, 489, 524, 522, 480, 533, 507, 485, 465, 492, 474, 498, 542, 505, 504, 475, 496, 547, 534, 508, 518, 497, 503, 518, 494, 551, 570, 491, 502, 486, 507, 572, 511, 554, 528, 500, 510, 524, 507, 516, 524, 517, 550, 531, 543, 526, 479, 500, 505, 489, 497, 518, 474, 525, 520, 523, 521, 508, 501, 539, 507, 525, 538, 523, 519, 503, 479, 503, 497, 523, 494, 515, 481, 514, 513, 476, 481, 467, 524, 501, 504, 509, 534, 524, 521, 527, 534, 516, 522, 545, 516, 501, 487, 508, 541, 519, 534, 502, 540, 506, 523, 491, 531, 500, 530, 542, 538, 487, 509, 488, 523, 517, 507, 487, 499] payload_rows_per_dst=[10910, 10908, 11178, 10916, 11006, 10832, 11004, 10998] dup_rows_per_dst=[5440, 5210, 5442, 5520, 5440, 5240, 5670, 5358] combine_rows_per_src=[10910, 10908, 11178, 10916, 11006, 10832, 11004, 10998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=458800 combine_payload_bytes=627785728 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628244528 ingress_bytes=778780672 link_bytes=778780672 token_bytes=1256030256 weight_movement_bytes=150994944 total_nvlink_bytes=1407025200 weight_to_token_ratio=0.120216 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876035138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627752960 dispatch_meta_bytes=459608 combine_payload_bytes=627752960 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628212568 ingress_bytes=703250432 link_bytes=703250432 token_bytes=1255965528 weight_movement_bytes=75497472 total_nvlink_bytes=1331463000 weight_to_token_ratio=0.060111 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918205378e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87366 payload_rows_remote=76502 dup_rows=43706 dup_rows_remote=38174 dedup_ratio=0.333450 router_tokens_per_expert=[506, 497, 478, 522, 481, 480, 509, 505, 498, 516, 534, 475, 535, 518, 509, 536, 487, 529, 527, 518, 494, 539, 501, 482, 538, 529, 494, 518, 516, 511, 537, 500, 546, 500, 491, 519, 494, 516, 564, 503, 532, 520, 528, 527, 504, 505, 537, 497, 491, 541, 508, 527, 504, 531, 487, 511, 504, 551, 514, 485, 505, 544, 492, 501, 507, 506, 512, 515, 483, 503, 553, 545, 516, 533, 529, 501, 494, 493, 519, 477, 478, 525, 508, 500, 537, 533, 486, 512, 486, 554, 515, 505, 518, 557, 498, 486, 504, 513, 489, 556, 502, 500, 502, 507, 541, 524, 524, 510, 477, 496, 538, 538, 509, 528, 507, 489, 503, 499, 528, 494, 498, 510, 498, 496, 493, 489, 559, 533] payload_rows_per_dst=[10848, 10926, 10996, 10982, 11016, 10864, 10988, 10746] dup_rows_per_dst=[5350, 5514, 5570, 5410, 5356, 5532, 5454, 5520] combine_rows_per_src=[10848, 10926, 10996, 10982, 11016, 10864, 10988, 10746] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=458744 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627212280 ingress_bytes=928743424 link_bytes=928743424 token_bytes=1253965816 weight_movement_bytes=301989888 total_nvlink_bytes=1555955704 weight_to_token_ratio=0.240828 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177777392e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=457728 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626883584 ingress_bytes=852918272 link_bytes=852918272 token_bytes=1253309440 weight_movement_bytes=226492416 total_nvlink_bytes=1479801856 weight_to_token_ratio=0.180715 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081620426e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87356 payload_rows_remote=76460 dup_rows=43716 dup_rows_remote=38332 dedup_ratio=0.333527 router_tokens_per_expert=[521, 524, 499, 509, 522, 492, 532, 529, 530, 455, 520, 501, 512, 545, 510, 537, 506, 500, 491, 507, 487, 554, 530, 571, 506, 521, 524, 509, 517, 522, 501, 545, 524, 516, 520, 546, 491, 556, 529, 521, 492, 503, 498, 522, 516, 489, 514, 478, 520, 564, 537, 466, 479, 488, 513, 521, 512, 510, 536, 535, 486, 494, 495, 498, 527, 550, 513, 540, 507, 498, 531, 503, 463, 504, 483, 488, 499, 489, 509, 521, 536, 529, 506, 485, 501, 527, 517, 500, 527, 539, 515, 485, 509, 486, 530, 509, 500, 490, 512, 489, 535, 522, 541, 539, 540, 491, 479, 507, 486, 492, 506, 511, 484, 488, 513, 514, 519, 538, 523, 503, 506, 487, 544, 491, 523, 509, 525, 505] payload_rows_per_dst=[10922, 11018, 11016, 10856, 10794, 10918, 10896, 10936] dup_rows_per_dst=[5554, 5564, 5414, 5452, 5456, 5484, 5384, 5408] combine_rows_per_src=[10922, 11018, 11016, 10856, 10794, 10918, 10896, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76418 dup_rows=43666 dup_rows_remote=38368 dedup_ratio=0.333145 router_tokens_per_expert=[476, 505, 535, 515, 528, 515, 481, 489, 472, 533, 484, 504, 525, 505, 536, 497, 516, 510, 513, 544, 503, 528, 502, 521, 546, 505, 499, 507, 535, 517, 525, 571, 528, 506, 489, 465, 499, 493, 520, 525, 512, 470, 509, 508, 550, 486, 519, 564, 556, 481, 499, 468, 480, 515, 499, 496, 496, 496, 543, 537, 504, 534, 526, 524, 522, 496, 516, 486, 526, 529, 528, 508, 495, 521, 495, 508, 491, 472, 519, 524, 487, 469, 495, 509, 559, 522, 482, 509, 524, 549, 516, 543, 524, 533, 515, 474, 519, 505, 521, 460, 526, 505, 507, 529, 470, 517, 527, 481, 513, 509, 498, 565, 514, 524, 556, 530, 508, 501, 515, 506, 565, 544, 478, 529, 517, 484, 516, 512] payload_rows_per_dst=[10782, 11068, 10988, 10878, 10972, 10864, 10892, 10962] dup_rows_per_dst=[5418, 5616, 5298, 5430, 5300, 5556, 5412, 5636] combine_rows_per_src=[10782, 11068, 10988, 10878, 10972, 10864, 10892, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=629506048 dispatch_meta_bytes=459816 combine_payload_bytes=629506048 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=629965864 ingress_bytes=855998464 link_bytes=855998464 token_bytes=1259471912 weight_movement_bytes=226492416 total_nvlink_bytes=1485964328 weight_to_token_ratio=0.179831 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085526543e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626704384 dispatch_meta_bytes=458704 combine_payload_bytes=626704384 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627163088 ingress_bytes=702201856 link_bytes=702201856 token_bytes=1253867472 weight_movement_bytes=75497472 total_nvlink_bytes=1329364944 weight_to_token_ratio=0.060212 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904907958e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626360320 dispatch_meta_bytes=459168 combine_payload_bytes=626360320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626819488 ingress_bytes=701857792 link_bytes=701857792 token_bytes=1253179808 weight_movement_bytes=75497472 total_nvlink_bytes=1328677280 weight_to_token_ratio=0.060245 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900544742e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626016256 dispatch_meta_bytes=459144 combine_payload_bytes=626016256 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626475400 ingress_bytes=701513728 link_bytes=701513728 token_bytes=1252491656 weight_movement_bytes=75497472 total_nvlink_bytes=1327989128 weight_to_token_ratio=0.060278 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.896181526e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76256 dup_rows=43738 dup_rows_remote=38194 dedup_ratio=0.333694 router_tokens_per_expert=[504, 501, 531, 519, 481, 551, 507, 530, 517, 545, 546, 492, 551, 527, 517, 475, 477, 510, 503, 481, 489, 557, 501, 479, 513, 488, 499, 515, 515, 556, 531, 510, 501, 496, 491, 530, 485, 520, 517, 503, 484, 495, 513, 538, 480, 517, 467, 508, 499, 524, 531, 498, 514, 520, 496, 518, 518, 558, 512, 532, 521, 516, 518, 536, 528, 462, 498, 519, 538, 521, 506, 497, 515, 529, 523, 491, 488, 512, 512, 494, 510, 487, 493, 526, 530, 490, 532, 536, 524, 515, 520, 503, 489, 535, 516, 519, 474, 500, 527, 518, 514, 521, 477, 508, 494, 529, 481, 522, 520, 503, 482, 479, 499, 495, 527, 535, 505, 491, 521, 545, 487, 560, 563, 518, 537, 520, 524, 528] payload_rows_per_dst=[10960, 10842, 10874, 11078, 10864, 10872, 10812, 11032] dup_rows_per_dst=[5628, 5406, 5216, 5544, 5402, 5578, 5286, 5678] combine_rows_per_src=[10960, 10842, 10874, 11078, 10864, 10872, 10812, 11032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87522 payload_rows_remote=76508 dup_rows=43550 dup_rows_remote=38182 dedup_ratio=0.332260 router_tokens_per_expert=[535, 467, 499, 512, 490, 481, 507, 514, 514, 540, 484, 520, 511, 542, 507, 499, 545, 499, 507, 493, 554, 523, 527, 522, 512, 495, 494, 554, 553, 488, 544, 533, 503, 524, 506, 500, 519, 485, 480, 481, 532, 530, 501, 490, 512, 490, 507, 514, 537, 540, 489, 514, 549, 522, 533, 471, 485, 482, 506, 500, 526, 552, 522, 500, 506, 477, 477, 523, 528, 504, 516, 521, 517, 528, 515, 532, 513, 521, 495, 518, 470, 472, 510, 507, 554, 546, 492, 529, 502, 553, 455, 543, 505, 516, 574, 489, 525, 493, 490, 514, 519, 508, 502, 498, 525, 540, 522, 544, 483, 530, 530, 505, 570, 542, 554, 495, 504, 510, 483, 502, 510, 505, 493, 492, 493, 509, 490, 481] payload_rows_per_dst=[10880, 11044, 10842, 11054, 11014, 10964, 10984, 10740] dup_rows_per_dst=[5364, 5642, 5306, 5402, 5368, 5470, 5472, 5526] combine_rows_per_src=[10880, 11044, 10842, 11054, 11014, 10964, 10984, 10740] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87256 payload_rows_remote=76266 dup_rows=43816 dup_rows_remote=38314 dedup_ratio=0.334290 router_tokens_per_expert=[515, 474, 524, 551, 511, 503, 495, 516, 532, 538, 533, 508, 533, 546, 520, 535, 523, 503, 535, 521, 478, 498, 467, 479, 479, 511, 512, 476, 477, 530, 480, 510, 524, 527, 483, 545, 524, 503, 496, 510, 504, 517, 496, 514, 519, 550, 539, 485, 526, 526, 509, 526, 503, 497, 534, 515, 515, 536, 529, 507, 476, 511, 512, 495, 502, 473, 485, 542, 503, 484, 504, 504, 520, 507, 515, 485, 503, 525, 525, 497, 552, 522, 546, 547, 516, 546, 444, 494, 511, 518, 482, 518, 544, 519, 558, 460, 481, 505, 535, 509, 525, 488, 524, 519, 478, 500, 535, 520, 505, 561, 485, 509, 511, 521, 529, 484, 516, 497, 540, 492, 534, 533, 502, 520, 519, 546, 507, 489] payload_rows_per_dst=[10888, 10742, 10600, 11314, 10878, 10738, 11106, 10990] dup_rows_per_dst=[5550, 5216, 5042, 5950, 5500, 5306, 5750, 5502] combine_rows_per_src=[10888, 10742, 10600, 11314, 10878, 10738, 11106, 10990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87494 payload_rows_remote=76508 dup_rows=43578 dup_rows_remote=37920 dedup_ratio=0.332474 router_tokens_per_expert=[516, 502, 544, 526, 515, 481, 529, 507, 543, 497, 523, 501, 514, 499, 508, 512, 497, 513, 512, 481, 484, 514, 522, 509, 516, 521, 514, 485, 508, 503, 499, 516, 478, 534, 569, 525, 481, 489, 540, 519, 494, 504, 527, 529, 534, 531, 479, 498, 529, 488, 501, 458, 494, 532, 504, 499, 511, 535, 481, 552, 543, 521, 470, 499, 500, 490, 506, 513, 488, 461, 545, 579, 528, 542, 553, 540, 527, 493, 497, 526, 550, 491, 487, 492, 533, 514, 490, 487, 503, 523, 517, 497, 490, 532, 519, 522, 503, 530, 518, 497, 512, 500, 561, 513, 515, 537, 506, 544, 497, 507, 519, 519, 497, 458, 516, 525, 515, 516, 515, 519, 485, 502, 496, 494, 546, 510, 533, 537] payload_rows_per_dst=[10986, 10722, 11002, 10866, 11110, 11014, 11012, 10782] dup_rows_per_dst=[5658, 5256, 5460, 5368, 5616, 5566, 5544, 5110] combine_rows_per_src=[10986, 10722, 11002, 10866, 11110, 11014, 11012, 10782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87646 payload_rows_remote=76642 dup_rows=43426 dup_rows_remote=37958 dedup_ratio=0.331314 router_tokens_per_expert=[502, 523, 510, 512, 518, 504, 504, 546, 483, 525, 529, 526, 480, 485, 517, 482, 544, 481, 523, 531, 504, 504, 548, 497, 484, 523, 527, 552, 513, 465, 535, 505, 522, 492, 516, 508, 511, 498, 518, 506, 527, 516, 518, 516, 549, 536, 482, 493, 546, 505, 516, 498, 536, 566, 496, 489, 493, 522, 526, 505, 509, 522, 492, 505, 516, 510, 506, 529, 519, 510, 493, 505, 531, 518, 522, 496, 525, 499, 493, 533, 530, 511, 553, 545, 460, 512, 536, 483, 506, 511, 521, 525, 466, 512, 490, 504, 467, 443, 552, 475, 506, 504, 483, 549, 520, 557, 550, 496, 530, 498, 574, 507, 502, 519, 481, 481, 502, 512, 489, 513, 565, 504, 491, 522, 468, 494, 585, 511] payload_rows_per_dst=[10918, 11004, 11080, 10890, 10968, 10928, 10946, 10912] dup_rows_per_dst=[5374, 5468, 5336, 5562, 5442, 5402, 5476, 5366] combine_rows_per_src=[10918, 11004, 11080, 10890, 10968, 10928, 10946, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=624689152 dispatch_meta_bytes=457800 combine_payload_bytes=624689152 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625146952 ingress_bytes=851181568 link_bytes=851181568 token_bytes=1249836104 weight_movement_bytes=226492416 total_nvlink_bytes=1476328520 weight_to_token_ratio=0.181218 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079418041e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=458760 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627212296 ingress_bytes=702251008 link_bytes=702251008 token_bytes=1253965832 weight_movement_bytes=75497472 total_nvlink_bytes=1329463304 weight_to_token_ratio=0.060207 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905531274e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87122 payload_rows_remote=76110 dup_rows=43950 dup_rows_remote=38272 dedup_ratio=0.335312 router_tokens_per_expert=[499, 532, 516, 544, 528, 498, 504, 517, 504, 543, 495, 491, 547, 511, 542, 471, 512, 463, 482, 502, 542, 536, 529, 490, 474, 508, 512, 494, 494, 544, 535, 497, 531, 524, 519, 466, 548, 512, 521, 491, 481, 512, 517, 535, 513, 496, 519, 563, 502, 510, 538, 544, 508, 500, 507, 498, 503, 492, 509, 512, 488, 480, 485, 519, 497, 529, 506, 538, 467, 467, 458, 480, 487, 552, 538, 503, 515, 503, 523, 459, 512, 495, 525, 529, 529, 546, 526, 541, 541, 513, 510, 502, 519, 540, 501, 516, 517, 495, 503, 477, 517, 529, 503, 529, 537, 514, 547, 497, 506, 500, 539, 515, 510, 527, 544, 507, 519, 545, 501, 531, 493, 515, 534, 520, 501, 483, 536, 479] payload_rows_per_dst=[10860, 10772, 10950, 10854, 10754, 11012, 10962, 10958] dup_rows_per_dst=[5624, 5456, 5546, 5336, 5290, 5678, 5488, 5532] combine_rows_per_src=[10860, 10772, 10950, 10854, 10754, 11012, 10962, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87166 payload_rows_remote=76386 dup_rows=43906 dup_rows_remote=38528 dedup_ratio=0.334976 router_tokens_per_expert=[537, 513, 521, 519, 536, 524, 494, 500, 525, 520, 515, 527, 518, 470, 512, 515, 533, 469, 534, 496, 515, 490, 542, 497, 525, 506, 510, 526, 513, 549, 483, 501, 497, 567, 503, 510, 505, 505, 494, 523, 535, 537, 501, 503, 531, 477, 472, 500, 489, 517, 488, 505, 517, 476, 490, 542, 499, 521, 544, 528, 482, 492, 492, 499, 534, 534, 513, 511, 515, 501, 526, 514, 504, 514, 500, 471, 511, 551, 512, 533, 535, 526, 477, 517, 515, 521, 569, 524, 492, 555, 507, 527, 506, 482, 563, 503, 475, 492, 500, 530, 524, 518, 508, 496, 512, 480, 501, 498, 536, 473, 513, 523, 501, 533, 524, 506, 492, 483, 525, 533, 529, 500, 516, 509, 523, 514, 535, 495] payload_rows_per_dst=[10974, 10870, 10936, 10814, 10838, 10974, 10780, 10980] dup_rows_per_dst=[5518, 5508, 5384, 5348, 5650, 5664, 5378, 5456] combine_rows_per_src=[10974, 10870, 10936, 10814, 10838, 10974, 10780, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624771072 dispatch_meta_bytes=458320 combine_payload_bytes=624771072 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625229392 ingress_bytes=775766016 link_bytes=775766016 token_bytes=1250000464 weight_movement_bytes=150994944 total_nvlink_bytes=1400995408 weight_to_token_ratio=0.120796 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.837805056e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87442 payload_rows_remote=76396 dup_rows=43630 dup_rows_remote=38192 dedup_ratio=0.332870 router_tokens_per_expert=[562, 520, 506, 526, 490, 488, 530, 508, 518, 483, 547, 559, 514, 526, 559, 517, 523, 507, 520, 500, 494, 452, 515, 507, 521, 501, 532, 517, 528, 545, 531, 538, 568, 495, 485, 535, 505, 500, 510, 498, 510, 515, 508, 509, 520, 523, 549, 512, 508, 497, 494, 501, 486, 518, 493, 500, 561, 502, 494, 556, 528, 515, 511, 509, 532, 514, 544, 468, 498, 529, 493, 523, 481, 501, 495, 467, 521, 526, 511, 512, 497, 503, 454, 475, 511, 510, 524, 482, 536, 534, 533, 536, 507, 548, 469, 487, 534, 545, 502, 488, 552, 510, 508, 502, 488, 511, 451, 537, 505, 503, 479, 536, 486, 530, 512, 516, 489, 522, 490, 500, 477, 539, 541, 525, 490, 498, 516, 534] payload_rows_per_dst=[10916, 10966, 11046, 10980, 10850, 10888, 10864, 10932] dup_rows_per_dst=[5790, 5496, 5438, 5366, 5380, 5324, 5438, 5398] combine_rows_per_src=[10916, 10966, 11046, 10980, 10850, 10888, 10864, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=457712 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627211248 ingress_bytes=702251008 link_bytes=702251008 token_bytes=1253964784 weight_movement_bytes=75497472 total_nvlink_bytes=1329462256 weight_to_token_ratio=0.060207 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905531274e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627851264 dispatch_meta_bytes=458400 combine_payload_bytes=627851264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628309664 ingress_bytes=703348736 link_bytes=703348736 token_bytes=1256160928 weight_movement_bytes=75497472 total_nvlink_bytes=1331658400 weight_to_token_ratio=0.060102 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919452011e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=623493120 dispatch_meta_bytes=457528 combine_payload_bytes=623493120 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=623950648 ingress_bytes=849985536 link_bytes=849985536 token_bytes=1247443768 weight_movement_bytes=226492416 total_nvlink_bytes=1473936184 weight_to_token_ratio=0.181565 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.077901304e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=459656 combine_payload_bytes=625754112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626213768 ingress_bytes=701251584 link_bytes=701251584 token_bytes=1251967880 weight_movement_bytes=75497472 total_nvlink_bytes=1327465352 weight_to_token_ratio=0.060303 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892857171e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458352 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626294384 ingress_bytes=776830976 link_bytes=776830976 token_bytes=1252130416 weight_movement_bytes=150994944 total_nvlink_bytes=1403125360 weight_to_token_ratio=0.120590 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851310248e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87358 payload_rows_remote=76468 dup_rows=43714 dup_rows_remote=38312 dedup_ratio=0.333511 router_tokens_per_expert=[491, 520, 498, 490, 513, 521, 497, 467, 490, 500, 493, 500, 487, 537, 528, 546, 486, 531, 487, 523, 485, 547, 520, 520, 509, 529, 503, 489, 493, 533, 523, 508, 516, 547, 522, 482, 505, 503, 532, 483, 515, 504, 558, 504, 534, 566, 513, 526, 475, 534, 508, 462, 521, 516, 494, 528, 509, 496, 498, 520, 538, 510, 536, 501, 552, 535, 534, 550, 556, 550, 513, 525, 493, 555, 500, 513, 550, 509, 479, 499, 480, 517, 505, 504, 517, 438, 531, 507, 530, 518, 516, 546, 497, 510, 512, 481, 522, 510, 481, 561, 464, 480, 481, 554, 491, 499, 517, 480, 494, 495, 512, 460, 543, 525, 533, 543, 516, 500, 526, 492, 523, 532, 495, 529, 509, 517, 535, 475] payload_rows_per_dst=[10884, 10932, 11048, 10890, 11028, 10906, 10704, 10966] dup_rows_per_dst=[5272, 5440, 5572, 5402, 5798, 5312, 5298, 5620] combine_rows_per_src=[10884, 10932, 11048, 10890, 11028, 10906, 10704, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87794 payload_rows_remote=76750 dup_rows=43278 dup_rows_remote=37938 dedup_ratio=0.330185 router_tokens_per_expert=[492, 510, 499, 502, 514, 512, 509, 482, 515, 520, 491, 529, 490, 512, 502, 524, 499, 493, 479, 543, 581, 493, 500, 518, 525, 555, 521, 489, 492, 455, 527, 525, 487, 498, 519, 536, 506, 515, 544, 499, 502, 527, 542, 504, 534, 544, 516, 550, 517, 511, 469, 563, 499, 532, 519, 520, 501, 528, 463, 485, 506, 478, 486, 474, 522, 473, 513, 522, 516, 527, 549, 481, 484, 535, 531, 494, 523, 501, 509, 512, 536, 522, 536, 471, 495, 533, 471, 526, 508, 501, 526, 511, 499, 531, 535, 533, 516, 518, 513, 524, 526, 504, 491, 534, 512, 519, 508, 550, 499, 496, 518, 482, 535, 502, 517, 539, 488, 554, 479, 503, 504, 527, 526, 509, 471, 511, 520, 543] payload_rows_per_dst=[10924, 11052, 11046, 10798, 11044, 10974, 10938, 11018] dup_rows_per_dst=[5282, 5338, 5600, 5304, 5340, 5494, 5482, 5438] combine_rows_per_src=[10924, 11052, 11046, 10798, 11044, 10974, 10938, 11018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87728 payload_rows_remote=76614 dup_rows=43344 dup_rows_remote=37750 dedup_ratio=0.330688 router_tokens_per_expert=[516, 507, 510, 529, 482, 500, 538, 504, 518, 483, 520, 470, 503, 494, 513, 503, 483, 486, 544, 498, 509, 508, 528, 546, 529, 501, 495, 533, 503, 523, 534, 486, 506, 534, 521, 508, 502, 485, 596, 496, 508, 531, 474, 500, 508, 507, 519, 528, 516, 516, 480, 508, 487, 529, 482, 504, 520, 497, 497, 527, 489, 539, 488, 479, 508, 512, 487, 514, 575, 482, 505, 516, 507, 495, 574, 521, 530, 530, 563, 505, 525, 470, 500, 487, 548, 520, 499, 536, 556, 490, 516, 496, 551, 495, 479, 504, 528, 503, 509, 524, 468, 499, 528, 475, 471, 534, 523, 518, 488, 526, 538, 484, 553, 526, 522, 529, 487, 559, 527, 536, 556, 477, 490, 530, 514, 493, 520, 528] payload_rows_per_dst=[10794, 11034, 10974, 10854, 11066, 10948, 10944, 11114] dup_rows_per_dst=[5278, 5378, 5472, 5262, 5652, 5420, 5288, 5594] combine_rows_per_src=[10794, 11034, 10974, 10854, 11066, 10948, 10944, 11114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76486 dup_rows=43590 dup_rows_remote=38288 dedup_ratio=0.332565 router_tokens_per_expert=[491, 503, 547, 509, 519, 501, 473, 515, 532, 488, 511, 497, 538, 504, 516, 505, 512, 492, 484, 482, 486, 484, 483, 509, 542, 510, 504, 519, 544, 534, 523, 480, 511, 496, 504, 526, 452, 530, 506, 528, 512, 531, 491, 504, 526, 527, 513, 513, 510, 519, 514, 494, 473, 497, 533, 512, 490, 517, 485, 542, 524, 546, 504, 472, 555, 511, 556, 483, 511, 504, 526, 510, 501, 503, 460, 496, 506, 522, 501, 511, 529, 514, 534, 522, 478, 541, 542, 501, 530, 498, 524, 524, 462, 505, 560, 526, 520, 488, 507, 519, 481, 506, 543, 516, 500, 538, 558, 491, 536, 492, 521, 515, 555, 559, 504, 515, 497, 547, 528, 547, 500, 504, 509, 514, 509, 501, 513, 518] payload_rows_per_dst=[10996, 11144, 10918, 10826, 10968, 10876, 10970, 10784] dup_rows_per_dst=[5302, 5668, 5530, 5438, 5344, 5596, 5492, 5220] combine_rows_per_src=[10996, 11144, 10918, 10826, 10968, 10876, 10970, 10784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87574 payload_rows_remote=76712 dup_rows=43498 dup_rows_remote=38154 dedup_ratio=0.331863 router_tokens_per_expert=[477, 491, 533, 475, 502, 510, 509, 537, 543, 492, 527, 567, 471, 504, 521, 520, 495, 519, 510, 520, 473, 511, 486, 529, 492, 489, 505, 522, 471, 528, 517, 536, 550, 508, 558, 477, 520, 509, 484, 521, 495, 528, 506, 489, 545, 536, 513, 446, 511, 484, 517, 509, 527, 485, 466, 508, 480, 497, 523, 491, 482, 484, 517, 533, 535, 500, 540, 534, 520, 526, 493, 474, 536, 510, 552, 545, 526, 518, 500, 501, 536, 611, 495, 489, 498, 511, 502, 512, 485, 495, 508, 540, 512, 535, 538, 518, 509, 526, 555, 507, 482, 556, 485, 521, 498, 515, 516, 520, 499, 521, 502, 485, 514, 549, 505, 546, 527, 517, 522, 530, 502, 457, 530, 517, 496, 522, 527, 502] payload_rows_per_dst=[11002, 10862, 10930, 10784, 11014, 11004, 10972, 11006] dup_rows_per_dst=[5356, 5344, 5440, 5244, 5606, 5566, 5422, 5520] combine_rows_per_src=[11002, 10862, 10930, 10784, 11014, 11004, 10972, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=459120 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626884976 ingress_bytes=701923328 link_bytes=701923328 token_bytes=1253310832 weight_movement_bytes=75497472 total_nvlink_bytes=1328808304 weight_to_token_ratio=0.060238 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901375831e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87400 payload_rows_remote=76516 dup_rows=43672 dup_rows_remote=38128 dedup_ratio=0.333191 router_tokens_per_expert=[516, 481, 498, 541, 502, 496, 508, 487, 537, 513, 540, 508, 486, 535, 500, 545, 536, 476, 528, 532, 517, 547, 521, 485, 489, 534, 519, 534, 524, 470, 503, 513, 532, 499, 492, 492, 505, 515, 511, 529, 502, 552, 514, 464, 500, 521, 513, 522, 542, 512, 529, 530, 519, 524, 522, 546, 510, 519, 512, 525, 517, 486, 526, 510, 502, 492, 482, 495, 507, 490, 494, 526, 531, 476, 533, 521, 505, 497, 501, 504, 523, 540, 489, 517, 512, 512, 484, 536, 516, 518, 516, 491, 523, 517, 529, 491, 544, 503, 476, 474, 513, 513, 512, 537, 506, 497, 497, 516, 528, 495, 517, 514, 518, 536, 509, 533, 553, 537, 468, 502, 505, 500, 504, 528, 501, 511, 518, 488] payload_rows_per_dst=[10970, 10998, 10812, 11006, 10856, 10884, 10878, 10996] dup_rows_per_dst=[5416, 5458, 5514, 5652, 5256, 5544, 5406, 5426] combine_rows_per_src=[10970, 10998, 10812, 11006, 10856, 10884, 10878, 10996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628736000 dispatch_meta_bytes=458752 combine_payload_bytes=628736000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629194752 ingress_bytes=704233472 link_bytes=704233472 token_bytes=1257930752 weight_movement_bytes=75497472 total_nvlink_bytes=1333428224 weight_to_token_ratio=0.060017 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930671709e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87386 payload_rows_remote=76470 dup_rows=43686 dup_rows_remote=38348 dedup_ratio=0.333298 router_tokens_per_expert=[506, 530, 525, 524, 547, 493, 490, 538, 499, 546, 489, 518, 485, 461, 499, 545, 539, 509, 541, 513, 524, 514, 534, 505, 493, 505, 511, 488, 532, 551, 484, 490, 521, 491, 506, 497, 519, 516, 510, 502, 459, 496, 502, 505, 506, 511, 483, 516, 509, 518, 500, 536, 510, 478, 504, 496, 536, 520, 533, 516, 542, 539, 504, 508, 528, 525, 524, 503, 522, 530, 468, 534, 495, 523, 544, 547, 507, 505, 473, 489, 522, 481, 535, 540, 530, 461, 507, 497, 529, 557, 543, 495, 479, 517, 527, 513, 501, 486, 517, 515, 527, 526, 530, 507, 494, 484, 486, 513, 489, 544, 500, 508, 506, 459, 511, 483, 510, 550, 539, 523, 499, 534, 530, 545, 527, 520, 486, 520] payload_rows_per_dst=[10904, 10888, 10834, 10906, 11068, 10918, 10916, 10952] dup_rows_per_dst=[5486, 5578, 5246, 5592, 5366, 5548, 5338, 5532] combine_rows_per_src=[10904, 10888, 10834, 10906, 11068, 10918, 10916, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627621888 dispatch_meta_bytes=457456 combine_payload_bytes=627621888 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628079344 ingress_bytes=854114304 link_bytes=854114304 token_bytes=1255701232 weight_movement_bytes=226492416 total_nvlink_bytes=1482193648 weight_to_token_ratio=0.180371 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083137163e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87408 payload_rows_remote=76518 dup_rows=43664 dup_rows_remote=38226 dedup_ratio=0.333130 router_tokens_per_expert=[517, 521, 542, 499, 508, 540, 539, 483, 493, 519, 543, 512, 515, 523, 516, 537, 479, 525, 549, 550, 502, 532, 486, 508, 516, 514, 508, 527, 499, 529, 541, 491, 512, 504, 503, 489, 525, 550, 501, 517, 511, 513, 536, 493, 529, 517, 476, 488, 489, 510, 495, 538, 496, 544, 474, 527, 514, 497, 533, 508, 508, 505, 544, 519, 483, 480, 516, 511, 513, 521, 532, 501, 518, 513, 501, 480, 516, 485, 503, 508, 477, 506, 524, 526, 514, 467, 558, 517, 528, 489, 494, 513, 483, 538, 521, 488, 487, 517, 492, 511, 504, 493, 527, 512, 505, 487, 530, 490, 527, 488, 529, 540, 441, 545, 516, 542, 504, 507, 492, 515, 491, 511, 510, 521, 562, 535, 525, 528] payload_rows_per_dst=[11012, 11052, 10890, 11008, 10752, 10896, 10836, 10962] dup_rows_per_dst=[5602, 5460, 5438, 5394, 5410, 5390, 5442, 5528] combine_rows_per_src=[11012, 11052, 10890, 11008, 10752, 10896, 10836, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626573312 dispatch_meta_bytes=459096 combine_payload_bytes=626573312 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=627032408 ingress_bytes=702070784 link_bytes=702070784 token_bytes=1253605720 weight_movement_bytes=75497472 total_nvlink_bytes=1329103192 weight_to_token_ratio=0.060224 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903245780e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628424704 dispatch_meta_bytes=459464 combine_payload_bytes=628424704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628884168 ingress_bytes=703922176 link_bytes=703922176 token_bytes=1257308872 weight_movement_bytes=75497472 total_nvlink_bytes=1332806344 weight_to_token_ratio=0.060047 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926724037e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626819072 dispatch_meta_bytes=458576 combine_payload_bytes=626819072 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627277648 ingress_bytes=853311488 link_bytes=853311488 token_bytes=1254096720 weight_movement_bytes=226492416 total_nvlink_bytes=1480589136 weight_to_token_ratio=0.180602 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082119079e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626442240 dispatch_meta_bytes=459272 combine_payload_bytes=626442240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626901512 ingress_bytes=701939712 link_bytes=701939712 token_bytes=1253343752 weight_movement_bytes=75497472 total_nvlink_bytes=1328841224 weight_to_token_ratio=0.060237 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901583603e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626835456 dispatch_meta_bytes=458976 combine_payload_bytes=626835456 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627294432 ingress_bytes=702332928 link_bytes=702332928 token_bytes=1254129888 weight_movement_bytes=75497472 total_nvlink_bytes=1329627360 weight_to_token_ratio=0.060199 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906570135e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87654 payload_rows_remote=76672 dup_rows=43418 dup_rows_remote=37836 dedup_ratio=0.331253 router_tokens_per_expert=[477, 511, 468, 511, 527, 553, 496, 497, 479, 515, 499, 502, 477, 515, 524, 478, 523, 510, 517, 508, 520, 510, 525, 499, 476, 490, 523, 514, 495, 505, 527, 550, 519, 518, 533, 522, 525, 536, 495, 531, 511, 480, 489, 501, 513, 524, 498, 512, 553, 516, 509, 554, 528, 494, 504, 477, 503, 529, 535, 507, 543, 493, 505, 532, 528, 514, 495, 506, 512, 528, 498, 507, 516, 511, 540, 499, 469, 504, 548, 526, 519, 523, 546, 485, 527, 498, 529, 500, 477, 492, 524, 498, 523, 503, 507, 510, 505, 523, 526, 477, 516, 528, 529, 500, 520, 498, 496, 515, 533, 492, 519, 495, 506, 498, 524, 508, 545, 552, 520, 522, 521, 528, 485, 549, 505, 546, 498, 485] payload_rows_per_dst=[10804, 11006, 10938, 10982, 11026, 10966, 10922, 11010] dup_rows_per_dst=[5254, 5378, 5476, 5582, 5376, 5356, 5422, 5574] combine_rows_per_src=[10804, 11006, 10938, 10982, 11026, 10966, 10922, 11010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87290 payload_rows_remote=76356 dup_rows=43782 dup_rows_remote=38320 dedup_ratio=0.334030 router_tokens_per_expert=[498, 500, 528, 497, 471, 518, 516, 526, 492, 515, 534, 541, 519, 511, 507, 533, 513, 505, 539, 469, 474, 522, 520, 536, 529, 494, 514, 489, 555, 521, 486, 561, 517, 539, 533, 498, 528, 508, 523, 495, 476, 473, 504, 527, 499, 511, 485, 555, 509, 509, 527, 519, 526, 487, 525, 533, 499, 520, 520, 492, 516, 512, 491, 489, 534, 541, 534, 536, 497, 489, 499, 495, 500, 560, 481, 530, 492, 495, 489, 526, 459, 482, 511, 522, 514, 548, 545, 546, 513, 539, 542, 545, 485, 505, 511, 526, 557, 515, 502, 504, 537, 486, 503, 464, 509, 527, 483, 494, 516, 549, 484, 533, 511, 461, 489, 517, 465, 523, 520, 509, 576, 526, 533, 500, 482, 495, 497, 500] payload_rows_per_dst=[11008, 10954, 10842, 10918, 10934, 10978, 10762, 10894] dup_rows_per_dst=[5404, 5500, 5500, 5430, 5462, 5608, 5564, 5314] combine_rows_per_src=[11008, 10954, 10842, 10918, 10934, 10978, 10762, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76014 dup_rows=43648 dup_rows_remote=37596 dedup_ratio=0.333008 router_tokens_per_expert=[477, 500, 559, 497, 535, 491, 488, 563, 517, 520, 519, 504, 499, 526, 520, 499, 523, 522, 534, 495, 493, 520, 548, 496, 497, 505, 514, 544, 535, 492, 553, 504, 535, 516, 551, 459, 529, 530, 511, 533, 548, 536, 498, 515, 503, 546, 495, 511, 501, 516, 515, 497, 521, 514, 529, 496, 496, 515, 498, 490, 524, 509, 514, 525, 534, 494, 517, 563, 523, 508, 545, 516, 510, 532, 503, 497, 499, 460, 497, 529, 465, 493, 498, 517, 502, 500, 512, 484, 538, 533, 514, 488, 507, 542, 473, 502, 485, 516, 524, 484, 499, 506, 549, 515, 539, 468, 497, 510, 474, 523, 479, 528, 497, 530, 530, 483, 523, 522, 521, 472, 508, 506, 517, 538, 519, 508, 532, 474] payload_rows_per_dst=[10980, 10528, 10522, 11222, 10660, 11056, 11046, 11410] dup_rows_per_dst=[5354, 4688, 4808, 6008, 5244, 5704, 5790, 6052] combine_rows_per_src=[10980, 10528, 10522, 11222, 10660, 11056, 11046, 11410] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628097024 dispatch_meta_bytes=458032 combine_payload_bytes=628097024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628555056 ingress_bytes=703594496 link_bytes=703594496 token_bytes=1256652080 weight_movement_bytes=75497472 total_nvlink_bytes=1332149552 weight_to_token_ratio=0.060078 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922568593e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87682 payload_rows_remote=76608 dup_rows=43390 dup_rows_remote=37682 dedup_ratio=0.331039 router_tokens_per_expert=[542, 542, 500, 501, 522, 501, 528, 483, 543, 544, 501, 604, 517, 488, 520, 555, 533, 477, 525, 489, 490, 511, 503, 530, 525, 527, 489, 515, 481, 463, 465, 497, 498, 494, 466, 505, 521, 514, 540, 520, 503, 476, 489, 574, 524, 528, 525, 494, 481, 526, 528, 515, 547, 486, 520, 534, 488, 484, 483, 521, 513, 490, 498, 531, 463, 514, 492, 510, 539, 507, 527, 505, 512, 506, 555, 517, 456, 516, 487, 508, 485, 503, 546, 520, 495, 570, 488, 491, 476, 538, 491, 534, 491, 504, 528, 518, 515, 547, 513, 524, 534, 491, 515, 489, 527, 542, 533, 519, 494, 468, 519, 480, 500, 507, 502, 537, 491, 510, 515, 521, 488, 508, 557, 564, 527, 535, 509, 536] payload_rows_per_dst=[11074, 10872, 11092, 10978, 10926, 10840, 11196, 10704] dup_rows_per_dst=[5708, 5310, 5674, 5480, 5302, 5206, 5764, 4946] combine_rows_per_src=[11074, 10872, 11092, 10978, 10926, 10840, 11196, 10704] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625508352 dispatch_meta_bytes=458704 combine_payload_bytes=625508352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625967056 ingress_bytes=701005824 link_bytes=701005824 token_bytes=1251475408 weight_movement_bytes=75497472 total_nvlink_bytes=1326972880 weight_to_token_ratio=0.060327 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889740589e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87556 payload_rows_remote=76484 dup_rows=43516 dup_rows_remote=38026 dedup_ratio=0.332001 router_tokens_per_expert=[525, 509, 508, 497, 493, 514, 522, 511, 518, 494, 530, 497, 511, 499, 545, 517, 514, 564, 500, 523, 512, 506, 547, 505, 546, 516, 493, 517, 508, 522, 493, 515, 520, 523, 493, 475, 486, 546, 471, 494, 541, 537, 525, 534, 528, 525, 514, 524, 531, 496, 487, 490, 519, 537, 500, 511, 515, 514, 526, 569, 534, 487, 467, 513, 528, 513, 466, 553, 533, 505, 509, 493, 472, 515, 520, 528, 534, 531, 543, 517, 500, 534, 502, 532, 473, 465, 499, 533, 511, 517, 526, 513, 523, 482, 526, 490, 521, 517, 477, 541, 519, 482, 545, 481, 526, 532, 497, 499, 521, 550, 501, 499, 516, 511, 541, 528, 483, 507, 484, 522, 460, 511, 499, 528, 492, 510, 486, 461] payload_rows_per_dst=[10924, 11072, 10984, 10862, 11016, 10936, 11038, 10724] dup_rows_per_dst=[5456, 5490, 5488, 5530, 5504, 5316, 5378, 5354] combine_rows_per_src=[10924, 11072, 10984, 10862, 11016, 10936, 11038, 10724] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76404 dup_rows=43648 dup_rows_remote=38388 dedup_ratio=0.333008 router_tokens_per_expert=[499, 486, 481, 528, 531, 526, 555, 539, 513, 530, 498, 552, 517, 509, 518, 477, 522, 514, 542, 499, 517, 530, 490, 513, 510, 517, 491, 491, 528, 514, 491, 498, 519, 500, 532, 521, 492, 498, 527, 533, 526, 500, 525, 506, 459, 536, 545, 534, 521, 506, 495, 459, 506, 521, 498, 464, 520, 477, 540, 509, 480, 528, 533, 485, 467, 559, 495, 539, 508, 502, 505, 479, 513, 501, 525, 531, 520, 497, 535, 524, 545, 519, 490, 517, 484, 530, 540, 484, 482, 490, 478, 512, 551, 526, 522, 528, 532, 522, 473, 501, 500, 501, 551, 544, 448, 493, 532, 480, 459, 519, 531, 554, 502, 520, 501, 538, 499, 517, 534, 519, 517, 489, 528, 510, 531, 524, 505, 543] payload_rows_per_dst=[10948, 10912, 10942, 10810, 10880, 10940, 11020, 10972] dup_rows_per_dst=[5570, 5422, 5496, 5274, 5520, 5456, 5260, 5650] combine_rows_per_src=[10948, 10912, 10942, 10810, 10880, 10940, 11020, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=622706688 dispatch_meta_bytes=454440 combine_payload_bytes=622706688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623161128 ingress_bytes=698204160 link_bytes=698204160 token_bytes=1245867816 weight_movement_bytes=75497472 total_nvlink_bytes=1321365288 weight_to_token_ratio=0.060598 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.854211545e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87464 payload_rows_remote=76650 dup_rows=43608 dup_rows_remote=38248 dedup_ratio=0.332703 router_tokens_per_expert=[493, 563, 548, 509, 490, 506, 522, 492, 531, 533, 492, 512, 511, 490, 506, 485, 529, 531, 534, 520, 503, 524, 524, 522, 527, 494, 511, 498, 468, 531, 495, 531, 528, 553, 558, 490, 504, 513, 500, 506, 512, 531, 498, 497, 554, 522, 508, 525, 501, 501, 536, 500, 515, 500, 475, 552, 506, 510, 520, 505, 493, 494, 526, 497, 561, 535, 524, 509, 530, 491, 520, 521, 507, 530, 507, 489, 526, 498, 497, 533, 526, 525, 492, 485, 519, 487, 491, 510, 519, 516, 531, 514, 500, 458, 515, 499, 498, 532, 524, 490, 508, 527, 499, 506, 517, 521, 517, 499, 508, 530, 509, 518, 549, 493, 507, 475, 488, 524, 457, 487, 497, 504, 533, 503, 490, 561, 508, 537] payload_rows_per_dst=[10934, 11036, 10970, 10910, 11046, 10814, 10878, 10876] dup_rows_per_dst=[5432, 5448, 5628, 5352, 5510, 5360, 5528, 5350] combine_rows_per_src=[10934, 11036, 10970, 10910, 11046, 10814, 10878, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76500 dup_rows=43582 dup_rows_remote=38176 dedup_ratio=0.332504 router_tokens_per_expert=[510, 500, 518, 534, 550, 520, 488, 515, 492, 520, 496, 541, 543, 494, 502, 510, 503, 514, 558, 534, 525, 518, 528, 538, 528, 506, 482, 573, 543, 516, 544, 513, 510, 476, 510, 476, 552, 518, 528, 536, 503, 526, 521, 507, 513, 486, 506, 530, 513, 489, 534, 520, 502, 506, 498, 515, 488, 487, 494, 525, 555, 542, 525, 542, 514, 514, 489, 540, 554, 564, 481, 538, 516, 512, 512, 505, 495, 507, 504, 523, 517, 527, 505, 515, 500, 526, 553, 528, 517, 504, 493, 501, 524, 506, 513, 473, 526, 516, 484, 514, 474, 499, 543, 508, 481, 496, 481, 476, 503, 488, 534, 520, 470, 502, 521, 466, 502, 517, 486, 518, 480, 482, 515, 533, 494, 481, 481, 486] payload_rows_per_dst=[11096, 11112, 10990, 10926, 10930, 10930, 10886, 10620] dup_rows_per_dst=[5370, 5734, 5406, 5544, 5606, 5474, 5200, 5248] combine_rows_per_src=[11096, 11112, 10990, 10926, 10930, 10930, 10886, 10620] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627572736 dispatch_meta_bytes=457160 combine_payload_bytes=627572736 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628029896 ingress_bytes=854065152 link_bytes=854065152 token_bytes=1255602632 weight_movement_bytes=226492416 total_nvlink_bytes=1482095048 weight_to_token_ratio=0.180385 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083074831e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=458040 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627014968 ingress_bytes=853049344 link_bytes=853049344 token_bytes=1253571896 weight_movement_bytes=226492416 total_nvlink_bytes=1480064312 weight_to_token_ratio=0.180678 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081786644e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625901568 dispatch_meta_bytes=459168 combine_payload_bytes=625901568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626360736 ingress_bytes=701399040 link_bytes=701399040 token_bytes=1252262304 weight_movement_bytes=75497472 total_nvlink_bytes=1327759776 weight_to_token_ratio=0.060289 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894727121e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=459592 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628376392 ingress_bytes=703414272 link_bytes=703414272 token_bytes=1256293192 weight_movement_bytes=75497472 total_nvlink_bytes=1331790664 weight_to_token_ratio=0.060095 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920283099e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626688000 dispatch_meta_bytes=458704 combine_payload_bytes=626688000 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627146704 ingress_bytes=853180416 link_bytes=853180416 token_bytes=1253834704 weight_movement_bytes=226492416 total_nvlink_bytes=1480327120 weight_to_token_ratio=0.180640 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081952862e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76562 dup_rows=43674 dup_rows_remote=38384 dedup_ratio=0.333206 router_tokens_per_expert=[522, 524, 522, 491, 512, 465, 498, 505, 525, 529, 511, 540, 446, 538, 499, 523, 481, 528, 527, 510, 502, 516, 535, 545, 508, 471, 505, 482, 493, 505, 506, 516, 496, 539, 536, 537, 552, 556, 533, 502, 523, 491, 475, 553, 493, 555, 503, 547, 507, 518, 497, 511, 510, 516, 488, 523, 498, 490, 501, 511, 516, 504, 506, 467, 562, 500, 518, 518, 527, 491, 517, 538, 533, 555, 509, 472, 537, 509, 496, 523, 520, 505, 522, 478, 535, 534, 528, 528, 512, 471, 552, 495, 484, 523, 529, 488, 528, 514, 502, 502, 548, 494, 469, 482, 524, 499, 470, 507, 528, 484, 533, 515, 478, 523, 533, 519, 511, 537, 510, 484, 513, 541, 492, 523, 507, 508, 523, 492] payload_rows_per_dst=[10826, 10882, 11090, 10836, 10976, 10974, 10904, 10910] dup_rows_per_dst=[5474, 5378, 5692, 5290, 5634, 5434, 5294, 5478] combine_rows_per_src=[10826, 10882, 11090, 10836, 10976, 10974, 10904, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87770 payload_rows_remote=76828 dup_rows=43302 dup_rows_remote=37844 dedup_ratio=0.330368 router_tokens_per_expert=[509, 537, 522, 533, 521, 528, 530, 476, 534, 493, 512, 502, 530, 500, 555, 491, 499, 530, 521, 538, 502, 524, 484, 490, 556, 503, 587, 528, 516, 501, 565, 503, 546, 500, 520, 502, 518, 521, 498, 503, 527, 471, 511, 536, 535, 502, 539, 485, 487, 480, 503, 507, 525, 483, 495, 555, 513, 526, 522, 449, 462, 500, 496, 522, 504, 506, 505, 542, 531, 526, 498, 528, 510, 512, 530, 513, 493, 517, 477, 508, 525, 503, 579, 502, 475, 516, 519, 501, 536, 532, 525, 550, 471, 488, 517, 508, 502, 527, 508, 524, 511, 502, 495, 536, 513, 526, 520, 512, 530, 456, 530, 487, 500, 497, 493, 481, 505, 546, 500, 511, 521, 503, 509, 524, 482, 479, 502, 498] payload_rows_per_dst=[11092, 11096, 10940, 10760, 10942, 11050, 10990, 10900] dup_rows_per_dst=[5454, 5598, 5488, 5290, 5458, 5444, 5368, 5202] combine_rows_per_src=[11092, 11096, 10940, 10760, 10942, 11050, 10990, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87822 payload_rows_remote=76674 dup_rows=43250 dup_rows_remote=37350 dedup_ratio=0.329971 router_tokens_per_expert=[511, 534, 525, 493, 525, 535, 520, 553, 530, 505, 509, 487, 547, 542, 493, 526, 549, 513, 546, 533, 522, 505, 505, 490, 536, 514, 513, 517, 518, 476, 514, 520, 464, 501, 522, 498, 480, 522, 573, 505, 493, 504, 503, 530, 507, 515, 471, 511, 539, 496, 536, 456, 502, 505, 484, 485, 511, 536, 517, 532, 519, 521, 496, 537, 528, 547, 538, 493, 483, 526, 561, 547, 555, 517, 444, 523, 536, 492, 511, 511, 529, 507, 490, 505, 493, 520, 454, 482, 503, 475, 494, 494, 501, 515, 521, 548, 497, 538, 525, 506, 496, 492, 499, 495, 504, 513, 529, 503, 495, 476, 532, 512, 532, 506, 530, 551, 524, 531, 494, 527, 483, 523, 524, 505, 498, 494, 492, 490] payload_rows_per_dst=[10868, 10786, 11002, 10970, 10974, 11058, 11016, 11148] dup_rows_per_dst=[5154, 5016, 5196, 5428, 5650, 5678, 5228, 5900] combine_rows_per_src=[10868, 10786, 11002, 10970, 10974, 11058, 11016, 11148] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=459784 combine_payload_bytes=627195904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627655688 ingress_bytes=702693376 link_bytes=702693376 token_bytes=1254851592 weight_movement_bytes=75497472 total_nvlink_bytes=1330349064 weight_to_token_ratio=0.060164 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911141123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=629374976 dispatch_meta_bytes=458688 combine_payload_bytes=629374976 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=629833664 ingress_bytes=855867392 link_bytes=855867392 token_bytes=1259208640 weight_movement_bytes=226492416 total_nvlink_bytes=1485701056 weight_to_token_ratio=0.179869 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085360325e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87572 payload_rows_remote=76694 dup_rows=43500 dup_rows_remote=38216 dedup_ratio=0.331879 router_tokens_per_expert=[533, 545, 487, 504, 521, 476, 488, 525, 511, 520, 487, 504, 498, 508, 505, 469, 550, 481, 484, 518, 483, 535, 508, 504, 500, 521, 553, 534, 546, 510, 518, 479, 524, 550, 533, 499, 522, 519, 513, 517, 511, 484, 562, 493, 530, 526, 500, 531, 541, 507, 498, 551, 476, 526, 531, 525, 511, 511, 486, 536, 463, 524, 529, 514, 492, 470, 498, 508, 550, 544, 504, 530, 485, 477, 538, 531, 536, 519, 512, 493, 535, 523, 506, 489, 512, 513, 491, 495, 474, 533, 494, 538, 550, 440, 498, 501, 522, 562, 533, 509, 507, 506, 528, 501, 533, 514, 503, 513, 480, 511, 512, 512, 489, 514, 581, 532, 517, 520, 489, 530, 459, 486, 536, 486, 512, 491, 490, 531] payload_rows_per_dst=[10878, 11096, 11008, 10908, 10908, 10914, 11016, 10844] dup_rows_per_dst=[5284, 5580, 5620, 5550, 5238, 5270, 5476, 5482] combine_rows_per_src=[10878, 11096, 11008, 10908, 10908, 10914, 11016, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76542 dup_rows=43528 dup_rows_remote=37978 dedup_ratio=0.332092 router_tokens_per_expert=[491, 500, 507, 505, 511, 522, 518, 498, 540, 535, 500, 530, 522, 502, 519, 529, 493, 513, 512, 551, 552, 488, 514, 510, 483, 509, 511, 530, 514, 498, 555, 543, 501, 501, 529, 520, 499, 511, 514, 468, 460, 520, 518, 571, 530, 498, 528, 489, 486, 545, 507, 486, 497, 489, 537, 527, 519, 525, 477, 518, 501, 488, 519, 544, 484, 489, 492, 501, 535, 506, 505, 477, 505, 509, 522, 495, 518, 551, 533, 516, 509, 486, 504, 547, 514, 508, 507, 514, 495, 526, 505, 535, 488, 510, 551, 536, 529, 516, 510, 487, 497, 495, 519, 521, 505, 487, 515, 524, 507, 559, 460, 542, 520, 487, 530, 515, 509, 496, 549, 497, 516, 473, 510, 499, 503, 504, 541, 514] payload_rows_per_dst=[10948, 11002, 10948, 10908, 10962, 10942, 10958, 10876] dup_rows_per_dst=[5510, 5550, 5366, 5422, 5314, 5528, 5388, 5450] combine_rows_per_src=[10948, 11002, 10948, 10908, 10962, 10942, 10958, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76670 dup_rows=43520 dup_rows_remote=38026 dedup_ratio=0.332031 router_tokens_per_expert=[481, 522, 497, 471, 523, 512, 547, 498, 493, 504, 493, 518, 481, 491, 519, 502, 547, 481, 498, 511, 553, 513, 498, 529, 542, 549, 540, 516, 517, 506, 528, 532, 463, 488, 541, 521, 525, 524, 485, 491, 503, 518, 521, 462, 529, 529, 527, 539, 468, 548, 556, 541, 489, 469, 513, 505, 537, 529, 518, 499, 505, 497, 510, 539, 535, 500, 526, 484, 479, 464, 524, 485, 521, 490, 527, 512, 531, 550, 477, 521, 541, 522, 506, 528, 554, 515, 498, 476, 510, 548, 538, 534, 477, 521, 507, 500, 519, 510, 495, 526, 512, 503, 480, 499, 547, 532, 532, 486, 509, 521, 504, 513, 528, 491, 512, 508, 492, 510, 534, 529, 484, 519, 477, 531, 489, 529, 531, 482] payload_rows_per_dst=[10818, 11020, 10902, 10948, 10966, 11004, 10882, 11012] dup_rows_per_dst=[5286, 5700, 5430, 5498, 5286, 5546, 5494, 5280] combine_rows_per_src=[10818, 11020, 10902, 10948, 10966, 11004, 10882, 11012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628113408 dispatch_meta_bytes=456096 combine_payload_bytes=628113408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628569504 ingress_bytes=703610880 link_bytes=703610880 token_bytes=1256682912 weight_movement_bytes=75497472 total_nvlink_bytes=1332180384 weight_to_token_ratio=0.060077 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922776366e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87722 payload_rows_remote=76818 dup_rows=43350 dup_rows_remote=37954 dedup_ratio=0.330734 router_tokens_per_expert=[497, 523, 496, 522, 476, 519, 528, 518, 539, 500, 524, 552, 539, 477, 514, 520, 545, 479, 520, 527, 501, 520, 476, 525, 482, 480, 499, 486, 521, 512, 471, 515, 518, 516, 497, 526, 501, 468, 502, 527, 509, 464, 535, 496, 509, 514, 499, 504, 529, 505, 479, 526, 532, 503, 500, 509, 535, 549, 516, 543, 511, 525, 516, 509, 574, 506, 483, 499, 483, 517, 512, 531, 544, 525, 490, 522, 528, 523, 521, 536, 535, 480, 502, 484, 536, 494, 523, 483, 467, 517, 518, 530, 537, 521, 544, 479, 540, 471, 554, 485, 505, 557, 540, 523, 535, 556, 516, 510, 512, 475, 487, 487, 510, 516, 535, 516, 496, 558, 521, 468, 502, 471, 512, 495, 535, 503, 528, 498] payload_rows_per_dst=[10996, 10832, 10994, 11008, 11074, 10904, 11002, 10912] dup_rows_per_dst=[5492, 5286, 5176, 5566, 5514, 5396, 5504, 5416] combine_rows_per_src=[10996, 10832, 10994, 11008, 11074, 10904, 11002, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76510 dup_rows=43582 dup_rows_remote=38042 dedup_ratio=0.332504 router_tokens_per_expert=[539, 511, 547, 515, 546, 538, 476, 475, 545, 535, 502, 532, 538, 517, 533, 501, 518, 515, 513, 521, 522, 480, 520, 499, 530, 540, 514, 481, 530, 543, 493, 500, 508, 532, 522, 521, 486, 508, 532, 499, 506, 518, 536, 537, 490, 512, 523, 530, 492, 498, 532, 538, 463, 479, 534, 493, 512, 538, 480, 530, 526, 514, 512, 503, 515, 509, 502, 487, 498, 491, 524, 498, 514, 541, 478, 508, 515, 524, 487, 499, 504, 495, 536, 493, 480, 488, 481, 475, 519, 508, 511, 534, 502, 474, 516, 521, 515, 519, 478, 532, 479, 520, 495, 489, 511, 478, 563, 504, 574, 518, 522, 509, 500, 525, 508, 503, 516, 535, 547, 509, 507, 530, 504, 540, 498, 505, 509, 494] payload_rows_per_dst=[11108, 10836, 10980, 10930, 10866, 10774, 10990, 11006] dup_rows_per_dst=[5592, 5602, 5540, 5358, 5314, 5300, 5422, 5454] combine_rows_per_src=[11108, 10836, 10980, 10930, 10866, 10774, 10990, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628277248 dispatch_meta_bytes=459640 combine_payload_bytes=628277248 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=628736888 ingress_bytes=703774720 link_bytes=703774720 token_bytes=1257014136 weight_movement_bytes=75497472 total_nvlink_bytes=1332511608 weight_to_token_ratio=0.060061 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924854087e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627032064 dispatch_meta_bytes=458080 combine_payload_bytes=627032064 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627490144 ingress_bytes=929021952 link_bytes=929021952 token_bytes=1254522208 weight_movement_bytes=301989888 total_nvlink_bytes=1556512096 weight_to_token_ratio=0.240721 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178130605e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628080640 dispatch_meta_bytes=458784 combine_payload_bytes=628080640 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628539424 ingress_bytes=779075584 link_bytes=779075584 token_bytes=1256620064 weight_movement_bytes=150994944 total_nvlink_bytes=1407615008 weight_to_token_ratio=0.120160 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.879775037e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=629293056 dispatch_meta_bytes=459088 combine_payload_bytes=629293056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629752144 ingress_bytes=704790528 link_bytes=704790528 token_bytes=1259045200 weight_movement_bytes=75497472 total_nvlink_bytes=1334542672 weight_to_token_ratio=0.059964 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.937735963e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626769920 dispatch_meta_bytes=458208 combine_payload_bytes=626769920 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627228128 ingress_bytes=702267392 link_bytes=702267392 token_bytes=1253998048 weight_movement_bytes=75497472 total_nvlink_bytes=1329495520 weight_to_token_ratio=0.060205 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905739047e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87374 payload_rows_remote=76568 dup_rows=43698 dup_rows_remote=38362 dedup_ratio=0.333389 router_tokens_per_expert=[514, 490, 502, 545, 512, 538, 512, 519, 498, 562, 530, 506, 535, 521, 500, 513, 514, 519, 493, 514, 537, 486, 536, 503, 518, 502, 527, 542, 477, 543, 512, 498, 484, 475, 551, 500, 530, 514, 515, 489, 472, 530, 543, 503, 514, 533, 543, 521, 493, 533, 498, 458, 511, 482, 475, 499, 495, 505, 555, 507, 491, 516, 523, 530, 475, 516, 533, 574, 494, 527, 504, 531, 512, 557, 524, 522, 480, 536, 537, 505, 503, 510, 487, 487, 505, 500, 536, 478, 542, 493, 491, 482, 496, 517, 493, 485, 518, 526, 510, 498, 532, 519, 541, 522, 505, 534, 513, 518, 509, 512, 542, 505, 506, 506, 533, 469, 489, 494, 512, 541, 514, 498, 494, 532, 477, 512, 477, 540] payload_rows_per_dst=[10988, 10884, 10990, 10806, 10996, 10778, 11028, 10904] dup_rows_per_dst=[5606, 5558, 5444, 5336, 5658, 5232, 5580, 5284] combine_rows_per_src=[10988, 10884, 10990, 10806, 10996, 10778, 11028, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87456 payload_rows_remote=76564 dup_rows=43616 dup_rows_remote=38184 dedup_ratio=0.332764 router_tokens_per_expert=[545, 515, 541, 506, 534, 539, 493, 481, 539, 499, 479, 488, 486, 525, 506, 512, 529, 492, 469, 487, 507, 527, 520, 514, 554, 562, 490, 493, 514, 508, 528, 506, 505, 519, 545, 498, 519, 543, 573, 526, 502, 528, 469, 522, 502, 520, 524, 474, 491, 513, 505, 571, 471, 486, 518, 551, 513, 529, 567, 527, 507, 499, 514, 503, 476, 489, 505, 485, 577, 492, 521, 493, 487, 530, 517, 481, 530, 544, 520, 515, 487, 505, 473, 559, 504, 500, 494, 496, 528, 524, 533, 496, 505, 501, 510, 542, 487, 535, 509, 505, 515, 530, 518, 496, 525, 506, 510, 513, 491, 478, 506, 513, 522, 504, 482, 521, 514, 534, 495, 495, 528, 523, 511, 512, 514, 477, 510, 516] payload_rows_per_dst=[10932, 10928, 10932, 11040, 10892, 10910, 10946, 10876] dup_rows_per_dst=[5444, 5472, 5606, 5490, 5432, 5404, 5328, 5440] combine_rows_per_src=[10932, 10928, 10932, 11040, 10892, 10910, 10946, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87374 payload_rows_remote=76474 dup_rows=43698 dup_rows_remote=38332 dedup_ratio=0.333389 router_tokens_per_expert=[511, 540, 483, 498, 515, 549, 532, 507, 526, 536, 501, 512, 514, 503, 513, 499, 501, 552, 446, 525, 543, 514, 543, 480, 551, 521, 516, 537, 502, 494, 528, 528, 508, 549, 523, 538, 506, 547, 497, 504, 500, 547, 485, 528, 522, 507, 510, 512, 486, 497, 525, 554, 514, 519, 550, 507, 532, 474, 497, 506, 512, 493, 494, 537, 500, 503, 505, 522, 529, 510, 524, 515, 518, 506, 508, 502, 536, 465, 525, 504, 533, 482, 492, 474, 517, 512, 550, 540, 506, 498, 503, 514, 487, 518, 516, 503, 502, 514, 509, 473, 526, 480, 475, 498, 466, 511, 547, 502, 525, 526, 522, 517, 502, 530, 499, 498, 503, 527, 494, 500, 553, 520, 488, 493, 541, 471, 512, 495] payload_rows_per_dst=[10906, 10964, 10986, 10866, 11044, 10916, 10792, 10900] dup_rows_per_dst=[5210, 5584, 5580, 5528, 5662, 5374, 5394, 5366] combine_rows_per_src=[10906, 10964, 10986, 10866, 11044, 10916, 10792, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627245056 dispatch_meta_bytes=459720 combine_payload_bytes=627245056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627704776 ingress_bytes=702742528 link_bytes=702742528 token_bytes=1254949832 weight_movement_bytes=75497472 total_nvlink_bytes=1330447304 weight_to_token_ratio=0.060160 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911764440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627212288 dispatch_meta_bytes=458992 combine_payload_bytes=627212288 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627671280 ingress_bytes=778207232 link_bytes=778207232 token_bytes=1254883568 weight_movement_bytes=150994944 total_nvlink_bytes=1405878512 weight_to_token_ratio=0.120326 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.868763111e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76546 dup_rows=43598 dup_rows_remote=38196 dedup_ratio=0.332626 router_tokens_per_expert=[519, 534, 490, 494, 529, 513, 531, 477, 523, 513, 529, 511, 491, 495, 495, 521, 544, 519, 548, 510, 496, 506, 488, 535, 547, 491, 555, 527, 482, 528, 515, 509, 500, 477, 518, 509, 538, 495, 520, 517, 492, 460, 461, 500, 531, 500, 487, 527, 508, 513, 544, 525, 518, 530, 479, 500, 533, 495, 532, 507, 510, 481, 487, 520, 519, 507, 534, 519, 530, 508, 508, 508, 500, 509, 534, 510, 534, 497, 507, 536, 518, 501, 489, 530, 536, 529, 494, 536, 493, 524, 534, 494, 542, 529, 527, 533, 510, 506, 507, 533, 513, 517, 493, 523, 491, 519, 503, 530, 543, 534, 479, 530, 486, 520, 488, 467, 537, 485, 485, 511, 513, 542, 530, 471, 507, 484, 495, 536] payload_rows_per_dst=[10928, 11076, 10946, 10868, 10846, 11094, 10890, 10826] dup_rows_per_dst=[5402, 5524, 5620, 5496, 5328, 5524, 5416, 5288] combine_rows_per_src=[10928, 11076, 10946, 10868, 10846, 11094, 10890, 10826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87528 payload_rows_remote=76586 dup_rows=43544 dup_rows_remote=38046 dedup_ratio=0.332214 router_tokens_per_expert=[506, 521, 490, 506, 531, 489, 538, 543, 511, 493, 515, 474, 508, 466, 473, 520, 507, 541, 524, 516, 508, 503, 521, 526, 483, 541, 518, 492, 522, 508, 511, 499, 485, 532, 509, 532, 516, 545, 506, 533, 543, 528, 481, 528, 509, 492, 547, 542, 507, 539, 505, 531, 524, 518, 520, 519, 509, 510, 503, 500, 539, 519, 531, 506, 470, 489, 529, 532, 542, 500, 492, 486, 511, 468, 538, 503, 491, 506, 523, 486, 522, 495, 567, 507, 519, 478, 513, 527, 536, 505, 531, 518, 505, 483, 491, 460, 477, 515, 497, 513, 526, 557, 511, 511, 542, 536, 512, 502, 461, 506, 498, 521, 504, 521, 478, 513, 507, 522, 501, 497, 553, 520, 525, 536, 514, 495, 476, 554] payload_rows_per_dst=[10882, 10942, 11008, 11026, 10864, 10942, 10832, 11032] dup_rows_per_dst=[5286, 5498, 5648, 5534, 5268, 5372, 5538, 5400] combine_rows_per_src=[10882, 10942, 11008, 11026, 10864, 10942, 10832, 11032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626475008 dispatch_meta_bytes=459224 combine_payload_bytes=626475008 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626934232 ingress_bytes=928464896 link_bytes=928464896 token_bytes=1253409240 weight_movement_bytes=301989888 total_nvlink_bytes=1555399128 weight_to_token_ratio=0.240935 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177424179e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87546 payload_rows_remote=76602 dup_rows=43526 dup_rows_remote=38044 dedup_ratio=0.332077 router_tokens_per_expert=[507, 519, 501, 517, 527, 530, 497, 553, 533, 541, 498, 560, 472, 552, 527, 511, 490, 522, 491, 512, 521, 491, 509, 556, 474, 477, 512, 509, 551, 496, 501, 509, 528, 491, 473, 483, 468, 475, 517, 489, 505, 529, 505, 522, 525, 519, 546, 474, 497, 496, 512, 529, 485, 482, 519, 519, 503, 533, 533, 513, 547, 535, 522, 520, 515, 491, 496, 526, 538, 563, 515, 481, 502, 518, 507, 480, 521, 502, 518, 507, 473, 492, 490, 508, 508, 581, 500, 535, 493, 501, 492, 516, 516, 464, 499, 511, 521, 534, 491, 518, 520, 545, 500, 504, 507, 529, 555, 482, 489, 516, 500, 502, 513, 501, 536, 501, 550, 493, 516, 538, 553, 503, 535, 483, 521, 506, 535, 520] payload_rows_per_dst=[11144, 10888, 10766, 10962, 10952, 10850, 10944, 11040] dup_rows_per_dst=[5546, 5354, 5332, 5528, 5408, 5308, 5482, 5568] combine_rows_per_src=[11144, 10888, 10766, 10962, 10952, 10850, 10944, 11040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87110 payload_rows_remote=76158 dup_rows=43962 dup_rows_remote=38360 dedup_ratio=0.335403 router_tokens_per_expert=[504, 506, 534, 511, 444, 539, 492, 515, 515, 508, 493, 527, 493, 504, 516, 526, 514, 469, 552, 532, 500, 539, 498, 485, 514, 520, 505, 505, 531, 498, 552, 520, 483, 532, 548, 504, 495, 506, 511, 525, 487, 497, 507, 527, 517, 512, 496, 507, 513, 537, 468, 544, 531, 516, 530, 508, 496, 474, 513, 457, 487, 497, 513, 520, 520, 534, 504, 528, 506, 519, 524, 541, 537, 540, 539, 531, 485, 527, 497, 500, 539, 486, 513, 526, 505, 542, 491, 506, 527, 588, 500, 517, 489, 527, 525, 496, 493, 524, 542, 512, 507, 506, 473, 524, 518, 515, 524, 521, 500, 544, 495, 517, 513, 488, 531, 504, 507, 488, 507, 506, 522, 489, 525, 517, 506, 481, 502, 507] payload_rows_per_dst=[10814, 10990, 10838, 10760, 11048, 10952, 10882, 10826] dup_rows_per_dst=[5440, 5478, 5470, 5448, 5616, 5602, 5548, 5360] combine_rows_per_src=[10814, 10990, 10838, 10760, 11048, 10952, 10882, 10826] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87216 payload_rows_remote=76356 dup_rows=43856 dup_rows_remote=38534 dedup_ratio=0.334595 router_tokens_per_expert=[518, 507, 570, 466, 494, 494, 523, 506, 522, 513, 481, 503, 483, 564, 512, 502, 496, 510, 491, 498, 512, 545, 477, 548, 560, 529, 510, 521, 523, 511, 487, 500, 538, 507, 479, 542, 484, 531, 477, 500, 513, 542, 499, 491, 510, 473, 511, 494, 526, 529, 509, 500, 527, 516, 506, 518, 485, 526, 509, 506, 507, 491, 493, 502, 489, 531, 480, 564, 513, 488, 540, 480, 503, 518, 517, 536, 515, 517, 487, 499, 507, 523, 545, 546, 542, 518, 487, 496, 520, 496, 517, 516, 526, 482, 517, 528, 501, 529, 504, 517, 493, 503, 481, 504, 508, 509, 540, 571, 508, 524, 507, 485, 482, 536, 552, 520, 545, 542, 521, 503, 534, 483, 530, 523, 510, 491, 526, 494] payload_rows_per_dst=[10856, 10948, 10860, 10886, 10950, 10864, 10910, 10942] dup_rows_per_dst=[5460, 5488, 5322, 5414, 5404, 5668, 5458, 5642] combine_rows_per_src=[10856, 10948, 10860, 10886, 10950, 10864, 10910, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627064832 dispatch_meta_bytes=458968 combine_payload_bytes=627064832 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627523800 ingress_bytes=778059776 link_bytes=778059776 token_bytes=1254588632 weight_movement_bytes=150994944 total_nvlink_bytes=1405583576 weight_to_token_ratio=0.120354 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866893162e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627392512 dispatch_meta_bytes=458528 combine_payload_bytes=627392512 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627851040 ingress_bytes=778387456 link_bytes=778387456 token_bytes=1255243552 weight_movement_bytes=150994944 total_nvlink_bytes=1406238496 weight_to_token_ratio=0.120291 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.871048605e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627523584 dispatch_meta_bytes=458584 combine_payload_bytes=627523584 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627982168 ingress_bytes=778518528 link_bytes=778518528 token_bytes=1255505752 weight_movement_bytes=150994944 total_nvlink_bytes=1406500696 weight_to_token_ratio=0.120266 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872710783e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=623886336 dispatch_meta_bytes=458072 combine_payload_bytes=623886336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624344408 ingress_bytes=699383808 link_bytes=699383808 token_bytes=1248230744 weight_movement_bytes=75497472 total_nvlink_bytes=1323728216 weight_to_token_ratio=0.060484 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.869171143e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76684 dup_rows=43506 dup_rows_remote=38206 dedup_ratio=0.331924 router_tokens_per_expert=[530, 516, 505, 512, 475, 537, 531, 489, 508, 538, 555, 503, 548, 509, 539, 487, 514, 520, 523, 535, 478, 471, 526, 513, 504, 522, 472, 507, 547, 536, 532, 497, 494, 508, 517, 500, 519, 526, 510, 504, 507, 486, 505, 532, 499, 491, 558, 501, 507, 531, 499, 491, 488, 519, 530, 476, 475, 532, 470, 509, 494, 525, 520, 525, 482, 472, 474, 519, 496, 525, 467, 508, 531, 540, 527, 480, 516, 473, 516, 539, 516, 491, 530, 525, 554, 545, 539, 546, 505, 500, 516, 499, 479, 484, 501, 483, 532, 512, 503, 511, 516, 532, 507, 501, 500, 507, 510, 496, 546, 548, 487, 511, 512, 527, 526, 487, 487, 523, 538, 541, 507, 518, 505, 549, 504, 525, 534, 529] payload_rows_per_dst=[11058, 10956, 10948, 10882, 10840, 10918, 10918, 11046] dup_rows_per_dst=[5506, 5438, 5366, 5300, 5290, 5508, 5520, 5578] combine_rows_per_src=[11058, 10956, 10948, 10882, 10840, 10918, 10918, 11046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625508352 dispatch_meta_bytes=459560 combine_payload_bytes=625508352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625967912 ingress_bytes=701005824 link_bytes=701005824 token_bytes=1251476264 weight_movement_bytes=75497472 total_nvlink_bytes=1326973736 weight_to_token_ratio=0.060327 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889740589e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76554 dup_rows=43558 dup_rows_remote=38288 dedup_ratio=0.332321 router_tokens_per_expert=[512, 502, 490, 528, 496, 551, 509, 540, 493, 493, 483, 533, 472, 505, 552, 505, 491, 551, 537, 510, 520, 489, 551, 487, 495, 487, 507, 463, 501, 528, 514, 545, 513, 495, 518, 483, 506, 519, 495, 519, 545, 516, 505, 541, 481, 529, 495, 523, 514, 556, 511, 524, 553, 499, 500, 512, 509, 494, 498, 526, 551, 533, 484, 487, 510, 515, 530, 512, 495, 515, 529, 493, 461, 520, 501, 536, 473, 509, 515, 501, 473, 520, 522, 508, 500, 526, 535, 448, 525, 501, 508, 519, 492, 509, 486, 534, 512, 522, 514, 515, 491, 542, 563, 534, 544, 507, 507, 541, 505, 515, 513, 552, 576, 494, 481, 518, 530, 486, 536, 506, 539, 499, 533, 524, 460, 510, 480, 492] payload_rows_per_dst=[10884, 10982, 10988, 10940, 10960, 10854, 11004, 10902] dup_rows_per_dst=[5444, 5370, 5378, 5562, 5270, 5358, 5750, 5426] combine_rows_per_src=[10884, 10982, 10988, 10940, 10960, 10854, 11004, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87336 payload_rows_remote=76458 dup_rows=43736 dup_rows_remote=38196 dedup_ratio=0.333679 router_tokens_per_expert=[489, 475, 523, 512, 491, 464, 511, 531, 526, 546, 509, 541, 516, 533, 510, 523, 548, 544, 495, 532, 554, 514, 545, 485, 506, 546, 493, 518, 518, 530, 543, 501, 532, 497, 498, 523, 499, 537, 510, 479, 508, 469, 486, 556, 516, 511, 503, 523, 504, 501, 546, 501, 483, 486, 513, 515, 482, 523, 528, 536, 508, 481, 576, 496, 518, 494, 519, 499, 488, 487, 530, 481, 531, 494, 511, 529, 517, 519, 535, 527, 488, 499, 499, 543, 524, 492, 516, 503, 508, 458, 528, 465, 489, 490, 536, 517, 528, 475, 488, 529, 520, 491, 489, 483, 508, 534, 512, 544, 533, 525, 513, 523, 541, 484, 495, 483, 508, 527, 504, 519, 563, 515, 512, 491, 507, 530, 535, 495] payload_rows_per_dst=[10876, 10630, 10896, 10884, 10976, 11246, 10950, 10878] dup_rows_per_dst=[5226, 5006, 5398, 5772, 5382, 5972, 5440, 5540] combine_rows_per_src=[10876, 10630, 10896, 10884, 10976, 11246, 10950, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628195328 dispatch_meta_bytes=459560 combine_payload_bytes=628195328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628654888 ingress_bytes=703692800 link_bytes=703692800 token_bytes=1256850216 weight_movement_bytes=75497472 total_nvlink_bytes=1332347688 weight_to_token_ratio=0.060069 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923815227e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627130368 dispatch_meta_bytes=459368 combine_payload_bytes=627130368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627589736 ingress_bytes=702627840 link_bytes=702627840 token_bytes=1254720104 weight_movement_bytes=75497472 total_nvlink_bytes=1330217576 weight_to_token_ratio=0.060171 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910310035e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87536 payload_rows_remote=76576 dup_rows=43536 dup_rows_remote=37950 dedup_ratio=0.332153 router_tokens_per_expert=[486, 465, 490, 483, 548, 516, 525, 538, 534, 552, 501, 498, 520, 532, 499, 520, 495, 496, 500, 506, 536, 517, 540, 531, 527, 508, 526, 504, 516, 531, 512, 528, 509, 526, 485, 540, 531, 511, 526, 511, 508, 509, 510, 496, 538, 535, 514, 484, 502, 554, 520, 516, 497, 502, 508, 521, 533, 503, 554, 521, 491, 500, 515, 483, 488, 511, 564, 520, 497, 495, 507, 557, 493, 517, 512, 543, 509, 476, 511, 510, 519, 491, 468, 532, 509, 510, 503, 546, 513, 493, 471, 521, 507, 468, 534, 510, 501, 501, 515, 488, 497, 499, 519, 529, 518, 540, 531, 530, 548, 502, 524, 512, 493, 489, 510, 528, 480, 537, 496, 505, 509, 509, 502, 493, 498, 492, 494, 509] payload_rows_per_dst=[10996, 10960, 10918, 10998, 10908, 10974, 10936, 10846] dup_rows_per_dst=[5418, 5586, 5548, 5442, 5512, 5216, 5572, 5242] combine_rows_per_src=[10996, 10960, 10918, 10998, 10908, 10974, 10936, 10846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76292 dup_rows=43674 dup_rows_remote=38144 dedup_ratio=0.333206 router_tokens_per_expert=[543, 513, 551, 500, 496, 535, 544, 518, 495, 533, 545, 511, 460, 526, 522, 502, 515, 496, 525, 538, 469, 526, 516, 498, 523, 485, 488, 516, 523, 487, 522, 493, 514, 516, 531, 510, 531, 498, 526, 515, 492, 522, 552, 527, 511, 503, 512, 491, 504, 475, 486, 491, 519, 487, 516, 513, 497, 572, 524, 517, 529, 531, 528, 514, 548, 548, 517, 515, 501, 480, 523, 510, 491, 508, 526, 540, 529, 497, 509, 506, 494, 513, 495, 504, 509, 533, 508, 520, 547, 513, 495, 535, 493, 510, 468, 482, 488, 515, 541, 522, 518, 521, 542, 500, 447, 503, 504, 509, 531, 525, 497, 516, 501, 479, 545, 489, 507, 531, 495, 521, 489, 491, 513, 510, 539, 469, 531, 512] payload_rows_per_dst=[11106, 10862, 11010, 10914, 10932, 10926, 10848, 10800] dup_rows_per_dst=[5530, 5330, 5572, 5522, 5564, 5660, 5252, 5244] combine_rows_per_src=[11106, 10862, 11010, 10914, 10932, 10926, 10848, 10800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626343936 dispatch_meta_bytes=458616 combine_payload_bytes=626343936 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626802552 ingress_bytes=852836352 link_bytes=852836352 token_bytes=1253146488 weight_movement_bytes=226492416 total_nvlink_bytes=1479638904 weight_to_token_ratio=0.180739 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081516540e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87084 payload_rows_remote=76254 dup_rows=43988 dup_rows_remote=38464 dedup_ratio=0.335602 router_tokens_per_expert=[495, 507, 489, 494, 567, 505, 502, 502, 498, 538, 505, 515, 503, 476, 524, 511, 511, 508, 510, 538, 516, 488, 517, 490, 517, 478, 540, 529, 537, 536, 539, 487, 478, 499, 506, 550, 529, 509, 500, 522, 528, 524, 538, 521, 550, 456, 513, 543, 525, 546, 502, 499, 502, 511, 494, 529, 490, 511, 507, 496, 556, 498, 506, 488, 521, 527, 480, 515, 526, 525, 524, 515, 522, 506, 535, 501, 492, 517, 514, 544, 524, 520, 507, 502, 498, 491, 516, 512, 522, 498, 496, 514, 473, 525, 475, 558, 542, 510, 494, 469, 520, 514, 497, 480, 529, 533, 516, 505, 536, 518, 498, 516, 470, 521, 516, 552, 487, 513, 541, 474, 514, 527, 467, 513, 538, 520, 505, 508] payload_rows_per_dst=[10884, 10890, 10854, 10858, 10918, 10958, 10830, 10892] dup_rows_per_dst=[5378, 5506, 5678, 5462, 5610, 5390, 5524, 5440] combine_rows_per_src=[10884, 10890, 10854, 10858, 10918, 10958, 10830, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87404 payload_rows_remote=76414 dup_rows=43668 dup_rows_remote=38164 dedup_ratio=0.333160 router_tokens_per_expert=[500, 539, 493, 501, 541, 548, 523, 518, 532, 518, 517, 549, 479, 506, 571, 479, 516, 583, 497, 529, 479, 531, 518, 518, 506, 509, 507, 506, 493, 521, 487, 474, 539, 497, 505, 538, 513, 467, 516, 544, 521, 483, 462, 498, 546, 521, 507, 512, 528, 462, 490, 543, 498, 504, 485, 506, 520, 495, 525, 520, 515, 507, 513, 480, 524, 501, 500, 538, 505, 505, 508, 513, 510, 535, 497, 513, 515, 511, 501, 516, 502, 524, 511, 526, 506, 519, 530, 514, 534, 542, 511, 508, 499, 516, 510, 495, 533, 516, 541, 507, 511, 503, 511, 512, 512, 523, 488, 506, 464, 526, 486, 483, 491, 470, 529, 489, 499, 495, 538, 538, 556, 505, 539, 503, 536, 503, 495, 541] payload_rows_per_dst=[11002, 10920, 10864, 10844, 10926, 10990, 10890, 10968] dup_rows_per_dst=[5626, 5428, 5474, 5338, 5458, 5504, 5354, 5486] combine_rows_per_src=[11002, 10920, 10864, 10844, 10926, 10990, 10890, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87610 payload_rows_remote=76712 dup_rows=43462 dup_rows_remote=38038 dedup_ratio=0.331589 router_tokens_per_expert=[565, 504, 522, 503, 530, 522, 516, 488, 537, 534, 521, 516, 483, 491, 484, 490, 538, 509, 527, 529, 527, 471, 537, 507, 508, 492, 535, 543, 485, 517, 526, 533, 496, 565, 513, 512, 491, 502, 526, 526, 483, 507, 499, 523, 516, 490, 493, 519, 485, 511, 507, 457, 516, 494, 538, 494, 532, 547, 518, 510, 488, 506, 527, 491, 462, 512, 496, 503, 480, 494, 522, 533, 488, 513, 542, 505, 522, 553, 510, 519, 516, 506, 482, 499, 507, 500, 472, 465, 515, 528, 477, 484, 554, 519, 497, 515, 526, 504, 490, 498, 515, 519, 471, 548, 527, 520, 530, 514, 495, 538, 501, 542, 535, 532, 548, 511, 547, 510, 503, 510, 516, 531, 502, 527, 513, 535, 515, 501] payload_rows_per_dst=[11022, 11008, 10898, 10894, 10948, 10800, 10936, 11104] dup_rows_per_dst=[5390, 5560, 5424, 5348, 5360, 5272, 5540, 5568] combine_rows_per_src=[11022, 11008, 10898, 10894, 10948, 10800, 10936, 11104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627310592 dispatch_meta_bytes=458104 combine_payload_bytes=627310592 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627768696 ingress_bytes=778305536 link_bytes=778305536 token_bytes=1255079288 weight_movement_bytes=150994944 total_nvlink_bytes=1406074232 weight_to_token_ratio=0.120307 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.870009744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=624984064 dispatch_meta_bytes=457744 combine_payload_bytes=624984064 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625441808 ingress_bytes=775979008 link_bytes=775979008 token_bytes=1250425872 weight_movement_bytes=150994944 total_nvlink_bytes=1401420816 weight_to_token_ratio=0.120755 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.840506094e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=624672768 dispatch_meta_bytes=458872 combine_payload_bytes=624672768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625131640 ingress_bytes=700170240 link_bytes=700170240 token_bytes=1249804408 weight_movement_bytes=75497472 total_nvlink_bytes=1325301880 weight_to_token_ratio=0.060407 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.879144207e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76484 dup_rows=43546 dup_rows_remote=38090 dedup_ratio=0.332230 router_tokens_per_expert=[488, 503, 484, 552, 531, 501, 457, 527, 502, 493, 496, 536, 523, 514, 521, 476, 507, 492, 447, 528, 543, 538, 515, 504, 541, 525, 535, 483, 539, 526, 532, 516, 509, 507, 552, 546, 480, 492, 541, 511, 529, 524, 497, 499, 502, 483, 486, 524, 490, 507, 531, 517, 527, 536, 523, 503, 546, 509, 464, 531, 466, 498, 542, 559, 496, 540, 507, 501, 523, 522, 492, 558, 518, 527, 487, 483, 531, 520, 523, 525, 496, 526, 534, 533, 499, 453, 471, 521, 474, 515, 520, 470, 527, 470, 504, 523, 513, 515, 496, 521, 551, 503, 514, 528, 490, 493, 511, 516, 480, 525, 505, 508, 503, 482, 498, 514, 517, 493, 545, 538, 492, 511, 487, 549, 497, 518, 585, 543] payload_rows_per_dst=[10916, 11008, 10986, 11042, 11042, 10732, 10818, 10982] dup_rows_per_dst=[5292, 5534, 5378, 5456, 5464, 5340, 5520, 5562] combine_rows_per_src=[10916, 11008, 10986, 11042, 11042, 10732, 10818, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625983488 dispatch_meta_bytes=458312 combine_payload_bytes=625983488 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626441800 ingress_bytes=701480960 link_bytes=701480960 token_bytes=1252425288 weight_movement_bytes=75497472 total_nvlink_bytes=1327922760 weight_to_token_ratio=0.060281 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895765982e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76396 dup_rows=43738 dup_rows_remote=38110 dedup_ratio=0.333694 router_tokens_per_expert=[553, 533, 525, 467, 527, 554, 538, 502, 532, 486, 496, 530, 518, 540, 502, 520, 544, 486, 520, 528, 519, 475, 508, 495, 534, 514, 519, 497, 503, 499, 489, 549, 501, 482, 459, 539, 495, 537, 479, 551, 497, 528, 541, 488, 538, 515, 511, 508, 488, 486, 506, 471, 519, 526, 510, 525, 546, 480, 489, 515, 518, 520, 566, 501, 548, 498, 467, 505, 526, 546, 505, 554, 502, 526, 526, 498, 484, 511, 542, 545, 493, 515, 489, 496, 508, 487, 515, 488, 506, 530, 519, 497, 498, 494, 511, 480, 548, 479, 491, 496, 492, 566, 538, 504, 500, 491, 512, 515, 495, 526, 538, 555, 487, 510, 519, 533, 477, 528, 519, 526, 540, 522, 480, 514, 519, 480, 515, 475] payload_rows_per_dst=[11062, 10878, 10848, 10962, 10938, 10774, 10968, 10904] dup_rows_per_dst=[5584, 5480, 5490, 5370, 5628, 5278, 5524, 5384] combine_rows_per_src=[11062, 10878, 10848, 10962, 10938, 10774, 10968, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628424704 dispatch_meta_bytes=459000 combine_payload_bytes=628424704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628883704 ingress_bytes=703922176 link_bytes=703922176 token_bytes=1257308408 weight_movement_bytes=75497472 total_nvlink_bytes=1332805880 weight_to_token_ratio=0.060047 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926724037e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87442 payload_rows_remote=76346 dup_rows=43630 dup_rows_remote=38150 dedup_ratio=0.332870 router_tokens_per_expert=[570, 535, 501, 524, 469, 537, 541, 497, 497, 516, 508, 512, 490, 536, 533, 505, 489, 471, 526, 488, 499, 492, 497, 497, 548, 526, 523, 526, 521, 497, 509, 520, 533, 538, 488, 492, 507, 464, 507, 517, 551, 527, 501, 531, 484, 507, 478, 483, 545, 514, 516, 517, 552, 519, 527, 522, 530, 516, 496, 522, 488, 543, 537, 530, 508, 523, 488, 497, 478, 490, 511, 552, 485, 488, 506, 492, 519, 562, 481, 558, 484, 535, 515, 521, 526, 475, 515, 539, 455, 513, 529, 524, 538, 530, 495, 509, 524, 520, 501, 494, 518, 518, 534, 505, 456, 493, 494, 515, 468, 525, 503, 517, 502, 514, 541, 534, 510, 510, 545, 534, 507, 500, 540, 483, 497, 517, 496, 498] payload_rows_per_dst=[10850, 10850, 10790, 11176, 10904, 10900, 10876, 11096] dup_rows_per_dst=[5408, 5280, 5426, 5856, 5372, 5506, 5302, 5480] combine_rows_per_src=[10850, 10850, 10790, 11176, 10904, 10900, 10876, 11096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=458296 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627015224 ingress_bytes=702054400 link_bytes=702054400 token_bytes=1253572152 weight_movement_bytes=75497472 total_nvlink_bytes=1329069624 weight_to_token_ratio=0.060226 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903038008e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458024 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626294056 ingress_bytes=852328448 link_bytes=852328448 token_bytes=1252130088 weight_movement_bytes=226492416 total_nvlink_bytes=1478622504 weight_to_token_ratio=0.180886 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080872446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625426432 dispatch_meta_bytes=457984 combine_payload_bytes=625426432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625884416 ingress_bytes=700923904 link_bytes=700923904 token_bytes=1251310848 weight_movement_bytes=75497472 total_nvlink_bytes=1326808320 weight_to_token_ratio=0.060335 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888701728e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87338 payload_rows_remote=76350 dup_rows=43734 dup_rows_remote=38158 dedup_ratio=0.333664 router_tokens_per_expert=[513, 486, 502, 507, 496, 517, 522, 498, 502, 501, 528, 540, 558, 495, 504, 522, 498, 531, 523, 560, 496, 486, 562, 503, 516, 508, 516, 517, 518, 530, 512, 506, 494, 507, 511, 512, 536, 489, 508, 501, 502, 495, 479, 515, 468, 474, 542, 519, 512, 506, 489, 532, 496, 542, 498, 507, 544, 526, 482, 482, 531, 503, 512, 517, 526, 548, 493, 516, 554, 539, 538, 510, 503, 461, 543, 467, 524, 542, 506, 476, 549, 497, 507, 521, 538, 520, 559, 543, 467, 537, 532, 512, 465, 513, 529, 480, 472, 553, 501, 525, 563, 518, 521, 507, 519, 493, 481, 491, 500, 542, 481, 468, 538, 467, 509, 511, 549, 494, 555, 488, 516, 502, 514, 505, 519, 518, 491, 506] payload_rows_per_dst=[10950, 10988, 10790, 10850, 10948, 10940, 10886, 10986] dup_rows_per_dst=[5432, 5576, 5314, 5508, 5544, 5598, 5384, 5378] combine_rows_per_src=[10950, 10988, 10790, 10850, 10948, 10940, 10886, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87438 payload_rows_remote=76504 dup_rows=43634 dup_rows_remote=38276 dedup_ratio=0.332901 router_tokens_per_expert=[501, 507, 498, 516, 515, 526, 512, 534, 541, 516, 550, 509, 508, 537, 510, 486, 494, 509, 538, 549, 509, 509, 518, 493, 520, 506, 516, 512, 499, 486, 513, 513, 514, 497, 494, 500, 526, 525, 511, 514, 505, 488, 527, 514, 530, 572, 489, 502, 517, 521, 478, 508, 510, 501, 518, 521, 527, 553, 529, 514, 520, 520, 487, 506, 516, 497, 478, 544, 510, 524, 494, 494, 512, 503, 539, 504, 478, 551, 514, 513, 472, 493, 519, 512, 561, 511, 536, 498, 526, 489, 540, 540, 489, 498, 509, 504, 513, 532, 465, 512, 503, 485, 505, 497, 498, 510, 460, 513, 521, 562, 501, 569, 494, 549, 528, 556, 517, 525, 527, 473, 502, 482, 491, 488, 487, 510, 510, 495] payload_rows_per_dst=[10980, 10932, 10972, 11006, 10930, 10850, 10934, 10834] dup_rows_per_dst=[5552, 5436, 5444, 5454, 5412, 5544, 5358, 5434] combine_rows_per_src=[10980, 10932, 10972, 11006, 10930, 10850, 10934, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87680 payload_rows_remote=76670 dup_rows=43392 dup_rows_remote=38002 dedup_ratio=0.331055 router_tokens_per_expert=[513, 550, 485, 534, 509, 481, 506, 481, 548, 476, 546, 518, 519, 518, 513, 502, 516, 528, 513, 502, 531, 522, 518, 510, 484, 538, 496, 514, 515, 521, 485, 497, 491, 502, 463, 515, 502, 512, 508, 544, 525, 526, 509, 502, 543, 531, 505, 484, 518, 534, 498, 508, 563, 492, 537, 478, 543, 502, 532, 517, 470, 544, 491, 474, 501, 506, 522, 569, 539, 470, 509, 495, 496, 492, 493, 498, 542, 496, 525, 532, 535, 512, 536, 515, 549, 523, 507, 517, 506, 475, 504, 513, 533, 553, 530, 505, 488, 507, 526, 496, 513, 504, 506, 530, 514, 514, 481, 506, 482, 464, 556, 530, 567, 488, 520, 482, 528, 508, 509, 480, 489, 522, 511, 474, 557, 505, 519, 510] payload_rows_per_dst=[11010, 11072, 10992, 10940, 10744, 11058, 10956, 10908] dup_rows_per_dst=[5390, 5606, 5426, 5462, 5258, 5568, 5314, 5368] combine_rows_per_src=[11010, 11072, 10992, 10940, 10744, 11058, 10956, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87500 payload_rows_remote=76702 dup_rows=43572 dup_rows_remote=38298 dedup_ratio=0.332428 router_tokens_per_expert=[521, 497, 508, 528, 499, 514, 452, 488, 510, 511, 484, 509, 501, 550, 521, 532, 491, 441, 535, 489, 521, 533, 538, 543, 503, 499, 525, 531, 524, 491, 486, 505, 516, 515, 520, 460, 505, 513, 516, 510, 464, 507, 470, 525, 520, 503, 498, 494, 518, 536, 496, 516, 518, 508, 528, 521, 469, 485, 495, 502, 528, 507, 482, 528, 554, 564, 502, 497, 540, 509, 495, 572, 496, 524, 528, 520, 511, 528, 542, 468, 553, 555, 506, 467, 511, 497, 544, 527, 480, 514, 517, 496, 512, 491, 490, 526, 547, 482, 555, 543, 499, 467, 506, 539, 528, 508, 491, 504, 518, 523, 516, 525, 469, 546, 534, 525, 532, 527, 493, 507, 489, 498, 581, 487, 493, 566, 534, 515] payload_rows_per_dst=[10920, 10978, 10798, 10870, 10996, 10878, 11086, 10974] dup_rows_per_dst=[5330, 5332, 5274, 5404, 5704, 5494, 5416, 5618] combine_rows_per_src=[10920, 10978, 10798, 10870, 10996, 10878, 11086, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87480 payload_rows_remote=76612 dup_rows=43592 dup_rows_remote=38270 dedup_ratio=0.332581 router_tokens_per_expert=[517, 502, 534, 495, 502, 508, 517, 484, 474, 477, 530, 499, 518, 553, 512, 455, 527, 541, 541, 560, 514, 505, 487, 505, 531, 522, 522, 534, 495, 472, 534, 542, 508, 506, 535, 534, 490, 509, 497, 519, 543, 490, 545, 505, 575, 477, 501, 500, 502, 520, 487, 484, 489, 520, 512, 497, 500, 533, 499, 524, 518, 487, 542, 475, 539, 501, 495, 502, 521, 534, 502, 503, 533, 540, 489, 509, 511, 523, 524, 524, 487, 496, 525, 508, 519, 501, 520, 483, 488, 521, 489, 528, 478, 522, 516, 514, 506, 538, 480, 514, 511, 531, 523, 532, 495, 539, 458, 536, 514, 553, 479, 493, 498, 540, 495, 490, 534, 505, 513, 507, 500, 507, 501, 542, 527, 547, 504, 547] payload_rows_per_dst=[10804, 11028, 10980, 10832, 10996, 10868, 10928, 11044] dup_rows_per_dst=[5350, 5636, 5488, 5346, 5504, 5322, 5476, 5470] combine_rows_per_src=[10804, 11028, 10980, 10832, 10996, 10868, 10928, 11044] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625459200 dispatch_meta_bytes=458032 combine_payload_bytes=625459200 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=625917232 ingress_bytes=927449088 link_bytes=927449088 token_bytes=1251376432 weight_movement_bytes=301989888 total_nvlink_bytes=1553366320 weight_to_token_ratio=0.241326 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176135992e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87592 payload_rows_remote=76592 dup_rows=43480 dup_rows_remote=38080 dedup_ratio=0.331726 router_tokens_per_expert=[490, 503, 543, 493, 501, 552, 531, 508, 517, 496, 548, 503, 496, 502, 527, 503, 497, 541, 494, 491, 500, 504, 517, 522, 490, 529, 505, 523, 518, 502, 531, 511, 482, 504, 553, 508, 478, 531, 506, 482, 528, 501, 507, 539, 514, 497, 475, 518, 468, 534, 476, 521, 533, 534, 509, 499, 504, 528, 538, 510, 525, 520, 491, 510, 513, 497, 534, 474, 513, 512, 509, 498, 547, 505, 516, 530, 543, 536, 511, 523, 545, 475, 506, 540, 529, 514, 507, 521, 532, 517, 513, 507, 541, 506, 481, 527, 559, 523, 518, 523, 522, 531, 531, 508, 502, 486, 500, 524, 544, 559, 499, 497, 485, 522, 515, 497, 478, 481, 480, 480, 486, 500, 509, 511, 513, 549, 481, 490] payload_rows_per_dst=[10998, 10902, 10882, 11000, 10976, 10982, 11062, 10790] dup_rows_per_dst=[5428, 5448, 5364, 5400, 5546, 5540, 5590, 5164] combine_rows_per_src=[10998, 10902, 10882, 11000, 10976, 10982, 11062, 10790] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=459120 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627179888 ingress_bytes=777715712 link_bytes=777715712 token_bytes=1253900656 weight_movement_bytes=150994944 total_nvlink_bytes=1404895600 weight_to_token_ratio=0.120420 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.862529946e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87374 payload_rows_remote=76538 dup_rows=43698 dup_rows_remote=38166 dedup_ratio=0.333389 router_tokens_per_expert=[505, 469, 547, 525, 528, 533, 508, 499, 518, 487, 536, 524, 550, 544, 508, 511, 524, 543, 529, 517, 489, 503, 499, 476, 525, 522, 536, 524, 467, 523, 481, 490, 464, 527, 522, 485, 499, 525, 547, 514, 507, 538, 500, 491, 511, 524, 505, 527, 513, 496, 477, 499, 520, 556, 486, 543, 459, 565, 505, 511, 496, 495, 519, 492, 515, 485, 515, 565, 491, 507, 541, 518, 508, 486, 490, 536, 495, 539, 496, 497, 508, 510, 494, 490, 509, 519, 491, 517, 496, 516, 552, 460, 486, 525, 515, 531, 500, 494, 523, 477, 512, 554, 514, 500, 524, 516, 545, 518, 512, 501, 496, 555, 499, 538, 503, 508, 537, 475, 542, 494, 522, 532, 473, 511, 512, 530, 514, 544] payload_rows_per_dst=[10970, 10932, 10942, 10816, 10836, 10894, 11024, 10960] dup_rows_per_dst=[5614, 5364, 5430, 5448, 5532, 5344, 5458, 5508] combine_rows_per_src=[10970, 10932, 10942, 10816, 10836, 10894, 11024, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628080640 dispatch_meta_bytes=458688 combine_payload_bytes=628080640 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628539328 ingress_bytes=779075584 link_bytes=779075584 token_bytes=1256619968 weight_movement_bytes=150994944 total_nvlink_bytes=1407614912 weight_to_token_ratio=0.120160 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.879775037e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628342784 dispatch_meta_bytes=460000 combine_payload_bytes=628342784 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628802784 ingress_bytes=703840256 link_bytes=703840256 token_bytes=1257145568 weight_movement_bytes=75497472 total_nvlink_bytes=1332643040 weight_to_token_ratio=0.060055 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925685176e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627605504 dispatch_meta_bytes=459528 combine_payload_bytes=627605504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628065032 ingress_bytes=703102976 link_bytes=703102976 token_bytes=1255670536 weight_movement_bytes=75497472 total_nvlink_bytes=1331168008 weight_to_token_ratio=0.060125 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916335428e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87620 payload_rows_remote=76520 dup_rows=43452 dup_rows_remote=37946 dedup_ratio=0.331512 router_tokens_per_expert=[495, 542, 484, 540, 501, 467, 513, 516, 522, 510, 498, 521, 536, 519, 502, 501, 518, 502, 527, 493, 447, 529, 514, 532, 523, 517, 488, 502, 487, 565, 507, 550, 522, 556, 482, 482, 495, 520, 526, 511, 516, 518, 482, 522, 544, 540, 497, 552, 566, 521, 511, 494, 536, 503, 470, 492, 490, 510, 554, 517, 489, 492, 492, 538, 519, 508, 524, 502, 516, 496, 519, 507, 497, 488, 547, 497, 493, 494, 527, 507, 517, 476, 513, 528, 520, 519, 503, 533, 521, 507, 518, 500, 548, 510, 532, 485, 511, 502, 493, 560, 510, 538, 489, 507, 529, 494, 493, 465, 495, 481, 526, 531, 476, 520, 532, 485, 514, 506, 558, 528, 520, 506, 508, 492, 521, 550, 510, 507] payload_rows_per_dst=[10886, 10988, 10986, 10696, 10706, 11222, 11036, 11100] dup_rows_per_dst=[5308, 5414, 5544, 5156, 4992, 5736, 5796, 5506] combine_rows_per_src=[10886, 10988, 10986, 10696, 10706, 11222, 11036, 11100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627441664 dispatch_meta_bytes=458688 combine_payload_bytes=627441664 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627900352 ingress_bytes=702939136 link_bytes=702939136 token_bytes=1255342016 weight_movement_bytes=75497472 total_nvlink_bytes=1330839488 weight_to_token_ratio=0.060141 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914257706e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626999296 dispatch_meta_bytes=458816 combine_payload_bytes=626999296 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627458112 ingress_bytes=777994240 link_bytes=777994240 token_bytes=1254457408 weight_movement_bytes=150994944 total_nvlink_bytes=1405452352 weight_to_token_ratio=0.120367 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866062073e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=457864 combine_payload_bytes=626851840 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627309704 ingress_bytes=777846784 link_bytes=777846784 token_bytes=1254161544 weight_movement_bytes=150994944 total_nvlink_bytes=1405156488 weight_to_token_ratio=0.120395 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864192123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76542 dup_rows=43514 dup_rows_remote=38148 dedup_ratio=0.331985 router_tokens_per_expert=[495, 504, 506, 485, 507, 557, 527, 511, 493, 550, 529, 516, 493, 537, 545, 557, 490, 500, 511, 489, 506, 502, 554, 539, 540, 513, 485, 517, 529, 487, 512, 517, 498, 499, 508, 511, 479, 495, 510, 511, 515, 550, 508, 488, 495, 516, 501, 499, 493, 521, 509, 488, 511, 492, 470, 509, 528, 533, 513, 535, 515, 528, 489, 511, 464, 486, 555, 534, 520, 489, 516, 507, 519, 544, 533, 487, 524, 550, 530, 519, 546, 520, 464, 472, 523, 514, 481, 549, 540, 521, 456, 521, 510, 547, 476, 525, 487, 516, 541, 511, 522, 530, 501, 490, 503, 513, 515, 520, 497, 512, 507, 548, 508, 564, 477, 506, 501, 522, 521, 486, 504, 513, 510, 455, 486, 507, 519, 571] payload_rows_per_dst=[11074, 11016, 10872, 10944, 10878, 10942, 10916, 10916] dup_rows_per_dst=[5550, 5366, 5294, 5346, 5676, 5388, 5510, 5384] combine_rows_per_src=[11074, 11016, 10872, 10944, 10878, 10942, 10916, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76616 dup_rows=43546 dup_rows_remote=38140 dedup_ratio=0.332230 router_tokens_per_expert=[498, 503, 509, 489, 539, 485, 506, 510, 565, 512, 505, 541, 502, 522, 470, 519, 495, 508, 525, 536, 518, 476, 535, 521, 474, 518, 477, 506, 543, 516, 534, 523, 525, 495, 500, 531, 502, 509, 527, 481, 489, 516, 518, 556, 490, 480, 540, 499, 514, 528, 533, 548, 533, 540, 537, 483, 499, 525, 525, 507, 524, 523, 516, 512, 518, 463, 492, 502, 493, 488, 552, 513, 520, 532, 518, 485, 526, 554, 512, 492, 473, 495, 499, 513, 527, 500, 496, 523, 498, 492, 487, 516, 504, 500, 468, 538, 490, 531, 500, 516, 476, 499, 502, 487, 487, 531, 491, 521, 504, 524, 500, 557, 521, 533, 557, 533, 527, 484, 510, 503, 520, 523, 525, 573, 528, 503, 512, 494] payload_rows_per_dst=[10896, 10970, 10910, 11148, 10830, 10832, 10868, 11072] dup_rows_per_dst=[5454, 5440, 5406, 5546, 5490, 5226, 5364, 5620] combine_rows_per_src=[10896, 10970, 10910, 11148, 10830, 10832, 10868, 11072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87658 payload_rows_remote=76732 dup_rows=43414 dup_rows_remote=37896 dedup_ratio=0.331223 router_tokens_per_expert=[537, 497, 489, 526, 512, 508, 523, 506, 551, 483, 524, 525, 514, 559, 507, 487, 475, 497, 495, 512, 471, 510, 514, 492, 533, 500, 524, 521, 503, 502, 517, 561, 540, 503, 504, 513, 511, 495, 519, 511, 526, 555, 539, 537, 511, 516, 476, 518, 481, 525, 493, 511, 494, 490, 509, 537, 541, 474, 516, 538, 493, 526, 512, 524, 491, 524, 496, 515, 505, 497, 511, 533, 493, 521, 525, 487, 503, 525, 486, 518, 523, 510, 511, 510, 502, 517, 514, 539, 484, 547, 514, 525, 513, 519, 505, 489, 507, 509, 504, 469, 551, 499, 500, 539, 466, 476, 519, 542, 540, 492, 497, 496, 508, 496, 541, 516, 516, 517, 496, 505, 554, 506, 478, 538, 520, 538, 539, 497] payload_rows_per_dst=[11016, 10876, 11054, 10890, 10934, 10926, 10932, 11030] dup_rows_per_dst=[5480, 5378, 5494, 5438, 5326, 5518, 5280, 5500] combine_rows_per_src=[11016, 10876, 11054, 10890, 10934, 10926, 10932, 11030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76558 dup_rows=43598 dup_rows_remote=38028 dedup_ratio=0.332626 router_tokens_per_expert=[511, 505, 531, 551, 513, 517, 535, 505, 486, 504, 514, 515, 471, 538, 488, 536, 519, 486, 516, 548, 506, 497, 502, 527, 463, 557, 528, 486, 534, 523, 500, 525, 492, 517, 509, 537, 511, 518, 506, 554, 527, 522, 524, 473, 510, 516, 575, 523, 501, 498, 477, 517, 534, 523, 480, 500, 512, 538, 496, 506, 519, 484, 484, 484, 495, 524, 496, 519, 478, 531, 493, 500, 526, 534, 556, 514, 472, 503, 486, 585, 487, 512, 510, 505, 505, 472, 532, 522, 559, 488, 486, 500, 488, 529, 543, 530, 538, 522, 554, 484, 534, 467, 479, 493, 520, 537, 498, 552, 493, 501, 511, 501, 532, 510, 510, 531, 498, 493, 508, 513, 506, 508, 561, 516, 509, 495, 494, 484] payload_rows_per_dst=[10916, 10946, 11320, 10836, 10696, 10968, 10928, 10864] dup_rows_per_dst=[5570, 5444, 5858, 5270, 5176, 5444, 5364, 5472] combine_rows_per_src=[10916, 10946, 11320, 10836, 10696, 10968, 10928, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87426 payload_rows_remote=76444 dup_rows=43646 dup_rows_remote=38156 dedup_ratio=0.332993 router_tokens_per_expert=[467, 511, 521, 516, 518, 524, 554, 542, 507, 495, 530, 522, 513, 563, 489, 535, 544, 495, 539, 488, 473, 521, 504, 481, 516, 488, 492, 531, 481, 557, 495, 492, 556, 517, 494, 488, 543, 517, 498, 501, 514, 510, 509, 518, 521, 533, 506, 493, 518, 501, 504, 515, 497, 496, 523, 514, 538, 518, 508, 502, 461, 524, 521, 537, 534, 522, 520, 529, 511, 474, 521, 546, 503, 493, 548, 493, 500, 508, 544, 505, 484, 500, 516, 501, 496, 480, 529, 491, 505, 506, 530, 512, 497, 524, 493, 512, 523, 538, 514, 528, 510, 512, 511, 514, 536, 522, 503, 476, 514, 503, 496, 536, 570, 497, 545, 524, 500, 517, 516, 508, 527, 491, 503, 501, 512, 479, 480, 504] payload_rows_per_dst=[11090, 10880, 10922, 10906, 10950, 10778, 10982, 10918] dup_rows_per_dst=[5524, 5314, 5514, 5448, 5552, 5374, 5490, 5430] combine_rows_per_src=[11090, 10880, 10922, 10906, 10950, 10778, 10982, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87312 payload_rows_remote=76292 dup_rows=43760 dup_rows_remote=38290 dedup_ratio=0.333862 router_tokens_per_expert=[522, 514, 534, 516, 528, 494, 507, 507, 506, 476, 500, 490, 493, 531, 481, 516, 479, 490, 551, 518, 516, 483, 538, 532, 534, 507, 469, 520, 480, 533, 506, 552, 510, 500, 501, 484, 522, 510, 537, 506, 505, 551, 487, 490, 506, 512, 544, 526, 502, 530, 537, 523, 514, 525, 485, 536, 517, 486, 508, 509, 542, 506, 522, 503, 533, 527, 531, 530, 560, 506, 534, 536, 509, 519, 469, 512, 501, 528, 516, 525, 492, 503, 489, 515, 532, 510, 531, 517, 499, 514, 572, 481, 493, 517, 510, 517, 546, 509, 528, 508, 520, 524, 477, 482, 530, 506, 481, 513, 516, 506, 499, 508, 517, 489, 515, 512, 529, 554, 511, 490, 511, 521, 489, 490, 469, 495, 496, 508] payload_rows_per_dst=[10884, 10902, 10868, 11020, 11072, 10874, 10870, 10822] dup_rows_per_dst=[5346, 5514, 5514, 5470, 5600, 5510, 5436, 5370] combine_rows_per_src=[10884, 10902, 10868, 11020, 11072, 10874, 10870, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87230 payload_rows_remote=76200 dup_rows=43842 dup_rows_remote=38260 dedup_ratio=0.334488 router_tokens_per_expert=[509, 520, 530, 553, 521, 529, 510, 506, 470, 514, 502, 543, 549, 511, 488, 483, 492, 532, 516, 524, 536, 520, 512, 505, 525, 499, 551, 510, 501, 509, 516, 493, 526, 542, 480, 544, 483, 461, 477, 522, 507, 501, 531, 535, 520, 523, 523, 540, 513, 523, 503, 488, 507, 515, 541, 491, 510, 538, 533, 493, 517, 487, 537, 537, 527, 494, 488, 552, 523, 506, 518, 520, 492, 550, 513, 560, 514, 486, 537, 526, 480, 489, 467, 474, 490, 551, 509, 486, 525, 530, 483, 525, 504, 493, 552, 496, 497, 512, 507, 472, 535, 480, 501, 506, 528, 512, 513, 493, 562, 544, 503, 488, 523, 507, 505, 516, 512, 509, 517, 535, 507, 488, 511, 495, 456, 504, 492, 519] payload_rows_per_dst=[10886, 10994, 10928, 10838, 11030, 10818, 10840, 10896] dup_rows_per_dst=[5590, 5488, 5502, 5628, 5582, 5290, 5466, 5296] combine_rows_per_src=[10886, 10994, 10928, 10838, 11030, 10818, 10840, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87666 payload_rows_remote=76646 dup_rows=43406 dup_rows_remote=37984 dedup_ratio=0.331161 router_tokens_per_expert=[498, 490, 529, 538, 514, 538, 486, 503, 520, 506, 540, 540, 488, 502, 494, 492, 477, 512, 516, 523, 501, 534, 517, 499, 502, 496, 495, 522, 512, 500, 529, 509, 463, 510, 510, 503, 511, 487, 488, 536, 514, 540, 555, 538, 472, 528, 494, 479, 535, 537, 518, 509, 501, 493, 561, 488, 507, 506, 478, 544, 511, 474, 468, 515, 538, 557, 490, 541, 502, 514, 531, 519, 520, 555, 522, 525, 563, 529, 525, 510, 454, 512, 509, 534, 503, 528, 531, 538, 517, 487, 510, 509, 520, 531, 512, 522, 504, 505, 510, 526, 516, 478, 492, 471, 465, 519, 540, 524, 519, 499, 528, 512, 503, 498, 536, 523, 494, 507, 558, 520, 480, 499, 503, 456, 539, 527, 500, 532] payload_rows_per_dst=[10986, 10874, 10718, 10960, 11108, 11110, 10890, 11020] dup_rows_per_dst=[5370, 5322, 5088, 5330, 5610, 5674, 5590, 5422] combine_rows_per_src=[10986, 10874, 10718, 10960, 11108, 11110, 10890, 11020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627032064 dispatch_meta_bytes=458760 combine_payload_bytes=627032064 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627490824 ingress_bytes=702529536 link_bytes=702529536 token_bytes=1254522888 weight_movement_bytes=75497472 total_nvlink_bytes=1330020360 weight_to_token_ratio=0.060180 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909063402e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627638272 dispatch_meta_bytes=459024 combine_payload_bytes=627638272 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628097296 ingress_bytes=778633216 link_bytes=778633216 token_bytes=1255735568 weight_movement_bytes=150994944 total_nvlink_bytes=1406730512 weight_to_token_ratio=0.120244 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874165188e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628588544 dispatch_meta_bytes=458512 combine_payload_bytes=628588544 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629047056 ingress_bytes=704086016 link_bytes=704086016 token_bytes=1257635600 weight_movement_bytes=75497472 total_nvlink_bytes=1333133072 weight_to_token_ratio=0.060031 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.928801759e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627163136 dispatch_meta_bytes=458344 combine_payload_bytes=627163136 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627621480 ingress_bytes=778158080 link_bytes=778158080 token_bytes=1254784616 weight_movement_bytes=150994944 total_nvlink_bytes=1405779560 weight_to_token_ratio=0.120335 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.868139795e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626229248 dispatch_meta_bytes=458400 combine_payload_bytes=626229248 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626687648 ingress_bytes=701726720 link_bytes=701726720 token_bytes=1252916896 weight_movement_bytes=75497472 total_nvlink_bytes=1328414368 weight_to_token_ratio=0.060257 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898882565e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=624984064 dispatch_meta_bytes=458328 combine_payload_bytes=624984064 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625442392 ingress_bytes=775979008 link_bytes=775979008 token_bytes=1250426456 weight_movement_bytes=150994944 total_nvlink_bytes=1401421400 weight_to_token_ratio=0.120755 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.840506094e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=624230400 dispatch_meta_bytes=457840 combine_payload_bytes=624230400 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=624688240 ingress_bytes=850722816 link_bytes=850722816 token_bytes=1248918640 weight_movement_bytes=226492416 total_nvlink_bytes=1475411056 weight_to_token_ratio=0.181351 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078836279e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627884032 dispatch_meta_bytes=458520 combine_payload_bytes=627884032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628342552 ingress_bytes=703381504 link_bytes=703381504 token_bytes=1256226584 weight_movement_bytes=75497472 total_nvlink_bytes=1331724056 weight_to_token_ratio=0.060099 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919867555e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15038828216 prefetch_bytes=830472192 reduce_bytes=981467136 weight_movement_bytes=1811939328 egress_bytes=7522160312 ingress_bytes=9328607232 link_bytes=9328607232 total_nvlink_bytes=16850767544 weight_to_token_ratio=0.120484 prefetch_share_of_ingress=0.089024 dedup_ratio=0.332249 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=13 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.858322559e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87730 payload_rows_remote=76738 dup_rows=43342 dup_rows_remote=37964 dedup_ratio=0.330673 router_tokens_per_expert=[514, 508, 510, 522, 504, 514, 523, 500, 499, 504, 494, 469, 520, 490, 536, 478, 518, 520, 502, 517, 515, 515, 542, 523, 512, 480, 489, 514, 501, 515, 483, 539, 505, 531, 538, 531, 519, 556, 519, 511, 502, 513, 520, 558, 510, 536, 514, 498, 534, 510, 483, 486, 495, 521, 455, 481, 492, 483, 515, 512, 515, 501, 544, 463, 535, 501, 511, 573, 503, 503, 495, 514, 509, 517, 509, 491, 484, 491, 509, 513, 544, 487, 551, 475, 487, 503, 488, 522, 490, 505, 506, 531, 468, 544, 495, 447, 568, 539, 534, 479, 552, 519, 522, 521, 552, 525, 525, 506, 516, 556, 498, 514, 519, 517, 522, 534, 521, 544, 501, 510, 520, 517, 503, 519, 516, 529, 480, 536] payload_rows_per_dst=[10816, 10992, 11114, 10802, 10940, 10946, 11134, 10986] dup_rows_per_dst=[5354, 5378, 5608, 5178, 5376, 5140, 5718, 5590] combine_rows_per_src=[10816, 10992, 11114, 10802, 10940, 10946, 11134, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87470 payload_rows_remote=76618 dup_rows=43602 dup_rows_remote=38074 dedup_ratio=0.332657 router_tokens_per_expert=[538, 487, 470, 548, 527, 490, 525, 505, 518, 522, 498, 501, 480, 529, 564, 553, 502, 549, 506, 526, 485, 527, 507, 511, 506, 548, 503, 501, 496, 534, 529, 497, 524, 492, 542, 515, 492, 497, 495, 525, 540, 538, 510, 518, 500, 479, 524, 499, 522, 531, 534, 501, 499, 504, 554, 498, 523, 489, 499, 507, 481, 467, 499, 505, 532, 522, 504, 515, 523, 494, 540, 521, 493, 473, 496, 511, 514, 551, 533, 501, 521, 505, 533, 524, 534, 499, 532, 546, 518, 512, 525, 503, 483, 517, 488, 482, 516, 556, 517, 473, 538, 555, 487, 468, 524, 567, 500, 509, 498, 517, 542, 455, 488, 528, 505, 518, 498, 508, 530, 536, 516, 501, 479, 507, 496, 492, 526, 456] payload_rows_per_dst=[11040, 11022, 10852, 10856, 10910, 11004, 10892, 10894] dup_rows_per_dst=[5470, 5432, 5528, 5370, 5536, 5440, 5552, 5274] combine_rows_per_src=[11040, 11022, 10852, 10856, 10910, 11004, 10892, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87378 payload_rows_remote=76624 dup_rows=43694 dup_rows_remote=38360 dedup_ratio=0.333359 router_tokens_per_expert=[487, 506, 508, 486, 521, 508, 502, 529, 478, 498, 524, 492, 499, 499, 487, 504, 485, 489, 501, 470, 527, 499, 535, 531, 517, 565, 541, 514, 488, 506, 512, 504, 509, 473, 516, 511, 552, 529, 529, 523, 538, 477, 491, 563, 491, 494, 473, 529, 511, 468, 538, 488, 484, 523, 546, 525, 511, 530, 506, 552, 495, 481, 548, 452, 522, 549, 486, 513, 543, 501, 506, 494, 545, 538, 504, 516, 490, 539, 484, 529, 533, 490, 488, 504, 495, 556, 504, 522, 556, 517, 531, 510, 544, 527, 513, 529, 527, 490, 535, 530, 506, 449, 536, 530, 481, 514, 468, 513, 483, 521, 527, 533, 517, 496, 516, 529, 515, 542, 522, 500, 500, 520, 524, 529, 504, 510, 516, 507] payload_rows_per_dst=[10754, 10958, 10966, 11038, 11022, 10962, 10814, 10864] dup_rows_per_dst=[5334, 5574, 5298, 5560, 5432, 5676, 5472, 5348] combine_rows_per_src=[10754, 10958, 10966, 11038, 11022, 10962, 10814, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87684 payload_rows_remote=76700 dup_rows=43388 dup_rows_remote=38030 dedup_ratio=0.331024 router_tokens_per_expert=[509, 475, 534, 487, 503, 494, 505, 484, 509, 501, 562, 520, 509, 501, 499, 502, 528, 536, 495, 520, 495, 548, 528, 524, 513, 519, 548, 547, 518, 473, 505, 533, 471, 514, 524, 505, 458, 524, 516, 527, 538, 501, 534, 501, 525, 520, 521, 563, 482, 521, 484, 507, 522, 492, 498, 481, 564, 497, 532, 502, 506, 494, 503, 542, 529, 511, 503, 545, 514, 511, 527, 533, 491, 498, 489, 507, 523, 538, 503, 490, 526, 524, 516, 487, 516, 509, 489, 484, 542, 502, 501, 520, 540, 484, 519, 512, 517, 523, 536, 495, 512, 557, 508, 498, 542, 525, 497, 493, 502, 515, 498, 460, 509, 494, 527, 535, 505, 495, 566, 522, 494, 505, 515, 488, 470, 519, 517, 521] payload_rows_per_dst=[10858, 11116, 10996, 10924, 10940, 10984, 10916, 10950] dup_rows_per_dst=[5330, 5544, 5488, 5330, 5484, 5358, 5440, 5414] combine_rows_per_src=[10858, 11116, 10996, 10924, 10940, 10984, 10916, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87532 payload_rows_remote=76378 dup_rows=43540 dup_rows_remote=37956 dedup_ratio=0.332184 router_tokens_per_expert=[480, 479, 500, 509, 533, 512, 492, 547, 452, 505, 477, 475, 523, 510, 534, 514, 507, 487, 514, 459, 536, 522, 511, 488, 523, 525, 563, 451, 505, 497, 530, 483, 456, 485, 517, 512, 499, 554, 552, 501, 520, 523, 466, 506, 498, 508, 552, 537, 521, 505, 511, 514, 545, 522, 502, 501, 529, 521, 507, 547, 525, 519, 471, 522, 498, 534, 522, 481, 525, 511, 507, 474, 512, 496, 510, 545, 496, 512, 519, 519, 465, 520, 540, 527, 503, 545, 489, 529, 495, 477, 562, 485, 512, 479, 516, 524, 522, 527, 495, 520, 505, 562, 547, 498, 538, 518, 533, 478, 501, 584, 531, 510, 511, 514, 510, 562, 494, 481, 519, 527, 546, 489, 490, 531, 528, 533, 510, 502] payload_rows_per_dst=[10798, 10908, 10920, 10984, 10868, 10850, 11154, 11050] dup_rows_per_dst=[5286, 5294, 5452, 5540, 5454, 5486, 5584, 5444] combine_rows_per_src=[10798, 10908, 10920, 10984, 10868, 10850, 11154, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87392 payload_rows_remote=76396 dup_rows=43680 dup_rows_remote=38334 dedup_ratio=0.333252 router_tokens_per_expert=[505, 502, 497, 538, 534, 467, 557, 523, 543, 513, 500, 529, 524, 519, 539, 494, 517, 540, 459, 510, 514, 511, 518, 512, 507, 502, 526, 499, 482, 569, 521, 464, 520, 472, 504, 498, 532, 521, 502, 524, 534, 517, 509, 533, 518, 565, 528, 499, 511, 502, 538, 521, 501, 543, 507, 515, 472, 493, 546, 538, 480, 483, 508, 513, 478, 515, 481, 516, 492, 498, 505, 497, 509, 503, 538, 442, 529, 507, 516, 475, 527, 522, 510, 514, 481, 513, 495, 551, 510, 480, 518, 507, 517, 496, 536, 504, 515, 521, 513, 507, 482, 497, 501, 540, 524, 509, 507, 542, 531, 495, 534, 505, 493, 505, 500, 522, 524, 498, 533, 520, 495, 532, 523, 539, 491, 532, 557, 485] payload_rows_per_dst=[10930, 10870, 10946, 10996, 10792, 10874, 10980, 11004] dup_rows_per_dst=[5638, 5432, 5606, 5346, 5210, 5488, 5466, 5494] combine_rows_per_src=[10930, 10870, 10946, 10996, 10792, 10874, 10980, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628637696 dispatch_meta_bytes=458808 combine_payload_bytes=628637696 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629096504 ingress_bytes=779632640 link_bytes=779632640 token_bytes=1257734200 weight_movement_bytes=150994944 total_nvlink_bytes=1408729144 weight_to_token_ratio=0.120053 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.886839291e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76364 dup_rows=43674 dup_rows_remote=38134 dedup_ratio=0.333206 router_tokens_per_expert=[520, 465, 536, 557, 498, 470, 503, 539, 502, 559, 541, 473, 508, 475, 535, 532, 517, 491, 533, 496, 515, 509, 566, 502, 486, 496, 497, 540, 494, 563, 517, 498, 550, 519, 521, 501, 505, 521, 506, 494, 547, 536, 493, 499, 521, 497, 502, 506, 482, 499, 513, 516, 548, 503, 486, 489, 514, 514, 535, 536, 482, 490, 525, 533, 534, 510, 527, 552, 537, 492, 505, 520, 478, 555, 497, 517, 501, 527, 525, 510, 522, 470, 462, 520, 491, 541, 523, 500, 522, 499, 510, 498, 505, 535, 506, 554, 500, 532, 524, 513, 495, 515, 554, 506, 503, 532, 522, 481, 481, 537, 542, 475, 496, 521, 511, 525, 491, 473, 562, 520, 504, 494, 476, 501, 479, 507, 503, 500] payload_rows_per_dst=[10892, 10932, 10936, 10922, 11034, 10922, 10962, 10798] dup_rows_per_dst=[5534, 5508, 5500, 5408, 5540, 5394, 5462, 5328] combine_rows_per_src=[10892, 10932, 10936, 10922, 11034, 10922, 10962, 10798] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627654656 dispatch_meta_bytes=458768 combine_payload_bytes=627654656 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628113424 ingress_bytes=854147072 link_bytes=854147072 token_bytes=1255768080 weight_movement_bytes=226492416 total_nvlink_bytes=1482260496 weight_to_token_ratio=0.180362 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083178718e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627703808 dispatch_meta_bytes=459936 combine_payload_bytes=627703808 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628163744 ingress_bytes=778698752 link_bytes=778698752 token_bytes=1255867552 weight_movement_bytes=150994944 total_nvlink_bytes=1406862496 weight_to_token_ratio=0.120232 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874996277e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628326400 dispatch_meta_bytes=458920 combine_payload_bytes=628326400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628785320 ingress_bytes=703823872 link_bytes=703823872 token_bytes=1257111720 weight_movement_bytes=75497472 total_nvlink_bytes=1332609192 weight_to_token_ratio=0.060056 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925477404e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458920 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626294952 ingress_bytes=701333504 link_bytes=701333504 token_bytes=1252130984 weight_movement_bytes=75497472 total_nvlink_bytes=1327628456 weight_to_token_ratio=0.060295 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893896032e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625688576 dispatch_meta_bytes=457336 combine_payload_bytes=625688576 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626145912 ingress_bytes=776683520 link_bytes=776683520 token_bytes=1251834488 weight_movement_bytes=150994944 total_nvlink_bytes=1402829432 weight_to_token_ratio=0.120619 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.849440298e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15064923776 prefetch_bytes=754974720 reduce_bytes=1132462080 weight_movement_bytes=1887436800 egress_bytes=7535214208 ingress_bytes=9417146368 link_bytes=9417146368 total_nvlink_bytes=16952360576 weight_to_token_ratio=0.125287 prefetch_share_of_ingress=0.080170 dedup_ratio=0.331935 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=15 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.951889298e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625573888 dispatch_meta_bytes=457992 combine_payload_bytes=625573888 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626031880 ingress_bytes=852066304 link_bytes=852066304 token_bytes=1251605768 weight_movement_bytes=226492416 total_nvlink_bytes=1478098184 weight_to_token_ratio=0.180961 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080540011e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15046576472 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=7526041944 ingress_bytes=8954986496 link_bytes=8954986496 total_nvlink_bytes=16481028440 weight_to_token_ratio=0.095334 prefetch_share_of_ingress=0.084308 dedup_ratio=0.333049 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=9 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044271 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.463486155e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15050733960 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=7528118664 ingress_bytes=8957067264 link_bytes=8957067264 total_nvlink_bytes=16485185928 weight_to_token_ratio=0.095308 prefetch_share_of_ingress=0.084288 dedup_ratio=0.332776 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=9 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.465685078e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15050343160 prefetch_bytes=905969664 reduce_bytes=301989888 weight_movement_bytes=1207959552 egress_bytes=7527924472 ingress_bytes=8730378240 link_bytes=8730378240 total_nvlink_bytes=16258302712 weight_to_token_ratio=0.080261 prefetch_share_of_ingress=0.103772 dedup_ratio=0.332684 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=4 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.226123752e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15037530856 prefetch_bytes=905969664 reduce_bytes=377487360 weight_movement_bytes=1283457024 egress_bytes=7521518312 ingress_bytes=8799469568 link_bytes=8799469568 total_nvlink_bytes=16320987880 weight_to_token_ratio=0.085350 prefetch_share_of_ingress=0.102957 dedup_ratio=0.333251 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=5 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.299138359e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15049029040 prefetch_bytes=754974720 reduce_bytes=1056964608 weight_movement_bytes=1811939328 egress_bytes=7527265712 ingress_bytes=9333702656 link_bytes=9333702656 total_nvlink_bytes=16860968368 weight_to_token_ratio=0.120402 prefetch_share_of_ingress=0.080887 dedup_ratio=0.332635 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=14 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044271 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.863707321e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87520 payload_rows_remote=76426 dup_rows=43552 dup_rows_remote=37876 dedup_ratio=0.332275 router_tokens_per_expert=[527, 544, 490, 480, 479, 509, 502, 507, 519, 504, 491, 543, 528, 550, 533, 528, 520, 526, 505, 520, 491, 515, 521, 520, 538, 502, 518, 529, 527, 490, 514, 475, 511, 500, 539, 499, 533, 477, 516, 527, 504, 491, 539, 521, 482, 516, 519, 499, 520, 527, 498, 495, 521, 506, 470, 500, 495, 508, 528, 520, 525, 477, 513, 500, 510, 540, 502, 486, 541, 524, 537, 504, 518, 523, 529, 521, 513, 518, 495, 525, 514, 537, 506, 513, 463, 499, 516, 533, 509, 516, 532, 502, 538, 532, 489, 502, 504, 487, 505, 466, 497, 471, 573, 501, 520, 501, 514, 534, 524, 491, 536, 502, 530, 536, 523, 494, 532, 500, 498, 482, 534, 517, 514, 479, 515, 539, 502, 507] payload_rows_per_dst=[11094, 11016, 10974, 11112, 10936, 10882, 10602, 10904] dup_rows_per_dst=[5676, 5432, 5346, 6014, 5334, 5520, 4730, 5500] combine_rows_per_src=[11094, 11016, 10974, 11112, 10936, 10882, 10602, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=4 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626081792 dispatch_meta_bytes=457208 combine_payload_bytes=626081792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626539000 ingress_bytes=701579264 link_bytes=701579264 token_bytes=1252620792 weight_movement_bytes=75497472 total_nvlink_bytes=1328118264 weight_to_token_ratio=0.060272 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897012615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=4 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15052533304 prefetch_bytes=603979776 reduce_bytes=1056964608 weight_movement_bytes=1660944384 egress_bytes=7529016888 ingress_bytes=9184460800 link_bytes=9184460800 total_nvlink_bytes=16713477688 weight_to_token_ratio=0.110343 prefetch_share_of_ingress=0.065761 dedup_ratio=0.332320 prefetch_slots=16 unused_prefetch_slots=368 slot_utilization=0.041667 reduce_slots=14 remote_demand=8 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=26924 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.705990921e-05 prefetch_bytes_per_gemm_flop=6.382761438e-06 anomalies=none + [2026-08-29 23:36:23] iteration 5/ 12 | consumed samples: 40 | elapsed time per iteration (ms): 356.2 | throughput per GPU (TFLOP/s/GPU): 507.6 | learning rate: 1.000000E-04 | global batch size: 8 | lm loss: 1.040502E+01 | loss scale: 1.0 | grad norm: 88.836 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87498 payload_rows_remote=76498 dup_rows=43574 dup_rows_remote=38146 dedup_ratio=0.332443 router_tokens_per_expert=[517, 520, 523, 553, 501, 483, 519, 480, 503, 518, 528, 516, 539, 526, 482, 512, 511, 519, 524, 506, 551, 521, 495, 461, 511, 524, 525, 493, 516, 511, 527, 519, 538, 539, 539, 535, 536, 563, 519, 499, 488, 513, 518, 519, 462, 510, 508, 522, 507, 560, 532, 524, 528, 507, 557, 514, 490, 525, 511, 528, 486, 477, 519, 507, 524, 506, 505, 485, 491, 525, 510, 502, 501, 510, 503, 494, 540, 490, 488, 501, 527, 537, 496, 528, 531, 526, 496, 522, 491, 512, 511, 508, 501, 518, 524, 516, 512, 477, 504, 530, 526, 524, 515, 485, 493, 506, 537, 489, 501, 497, 493, 477, 525, 551, 499, 505, 520, 535, 550, 527, 496, 481, 466, 475, 509, 493, 486, 519] payload_rows_per_dst=[11106, 11000, 11002, 10906, 10830, 11080, 10766, 10808] dup_rows_per_dst=[5334, 5428, 5614, 5638, 5320, 5408, 5366, 5466] combine_rows_per_src=[11106, 11000, 11002, 10906, 10830, 11080, 10766, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87530 payload_rows_remote=76588 dup_rows=43542 dup_rows_remote=38152 dedup_ratio=0.332199 router_tokens_per_expert=[505, 467, 492, 500, 506, 523, 546, 503, 514, 525, 516, 538, 487, 483, 530, 557, 500, 512, 537, 523, 536, 523, 469, 550, 491, 538, 473, 569, 469, 527, 520, 492, 562, 517, 531, 483, 486, 536, 531, 506, 514, 501, 534, 526, 508, 523, 443, 465, 556, 561, 503, 523, 495, 545, 509, 476, 488, 502, 532, 538, 519, 552, 537, 522, 508, 545, 510, 507, 496, 504, 512, 526, 520, 509, 515, 500, 497, 496, 520, 533, 475, 486, 513, 516, 503, 480, 520, 505, 489, 522, 509, 509, 513, 525, 540, 495, 535, 502, 505, 487, 493, 509, 481, 521, 529, 479, 523, 514, 471, 522, 515, 529, 538, 504, 515, 519, 521, 508, 512, 529, 512, 482, 536, 512, 493, 488, 515, 494] payload_rows_per_dst=[10888, 10980, 10942, 11100, 10898, 10798, 11004, 10920] dup_rows_per_dst=[5496, 5478, 5390, 5616, 5498, 5402, 5226, 5436] combine_rows_per_src=[10888, 10980, 10942, 11100, 10898, 10798, 11004, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87434 payload_rows_remote=76462 dup_rows=43638 dup_rows_remote=38236 dedup_ratio=0.332932 router_tokens_per_expert=[553, 517, 468, 560, 540, 519, 509, 503, 507, 535, 523, 487, 502, 548, 501, 506, 541, 533, 513, 539, 502, 523, 564, 453, 495, 518, 451, 497, 501, 530, 517, 529, 486, 502, 517, 488, 502, 498, 471, 507, 514, 529, 555, 554, 508, 505, 529, 535, 528, 511, 526, 532, 492, 502, 505, 519, 521, 476, 512, 496, 509, 523, 504, 501, 524, 536, 553, 510, 520, 520, 518, 499, 517, 527, 493, 561, 523, 491, 486, 515, 516, 520, 499, 517, 526, 521, 490, 548, 521, 474, 488, 481, 492, 481, 539, 491, 538, 528, 495, 500, 510, 518, 508, 515, 482, 492, 559, 478, 524, 512, 526, 502, 485, 523, 541, 518, 502, 511, 529, 504, 504, 486, 498, 497, 513, 517, 489, 494] payload_rows_per_dst=[10866, 10952, 10984, 10874, 10964, 10934, 10972, 10888] dup_rows_per_dst=[5486, 5460, 5416, 5440, 5622, 5478, 5402, 5334] combine_rows_per_src=[10866, 10952, 10984, 10874, 10964, 10934, 10972, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87864 payload_rows_remote=76840 dup_rows=43208 dup_rows_remote=37728 dedup_ratio=0.329651 router_tokens_per_expert=[546, 479, 500, 533, 536, 487, 494, 535, 502, 488, 559, 511, 531, 519, 548, 512, 506, 510, 498, 476, 496, 519, 500, 501, 524, 511, 512, 491, 487, 512, 545, 518, 496, 537, 503, 496, 524, 522, 467, 490, 505, 524, 530, 498, 534, 519, 530, 507, 514, 495, 548, 500, 539, 530, 511, 519, 508, 550, 497, 523, 526, 497, 471, 513, 521, 530, 537, 558, 488, 517, 532, 532, 523, 532, 460, 523, 532, 498, 554, 484, 496, 514, 487, 496, 532, 530, 486, 501, 517, 502, 509, 502, 479, 486, 488, 525, 504, 496, 523, 506, 509, 505, 498, 504, 550, 481, 486, 541, 541, 499, 531, 522, 506, 513, 521, 503, 528, 497, 549, 489, 484, 497, 549, 526, 468, 497, 510, 523] payload_rows_per_dst=[10592, 10896, 10962, 10922, 11120, 11262, 11086, 11024] dup_rows_per_dst=[4902, 5316, 5218, 5244, 5522, 5904, 5622, 5480] combine_rows_per_src=[10592, 10896, 10962, 10922, 11120, 11262, 11086, 11024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87608 payload_rows_remote=76710 dup_rows=43464 dup_rows_remote=38214 dedup_ratio=0.331604 router_tokens_per_expert=[470, 494, 545, 585, 521, 514, 522, 536, 528, 507, 549, 544, 502, 510, 448, 517, 516, 530, 523, 482, 495, 513, 506, 532, 512, 527, 507, 494, 502, 502, 513, 498, 499, 515, 509, 497, 517, 558, 558, 488, 525, 520, 537, 520, 542, 480, 514, 522, 532, 504, 547, 478, 522, 547, 531, 500, 503, 482, 508, 523, 476, 491, 508, 469, 568, 462, 481, 483, 528, 507, 508, 503, 508, 531, 562, 496, 529, 500, 501, 491, 500, 494, 525, 506, 519, 498, 526, 487, 499, 487, 485, 560, 484, 491, 506, 507, 477, 548, 497, 509, 548, 480, 513, 504, 530, 488, 532, 519, 505, 504, 493, 543, 519, 500, 547, 503, 510, 546, 544, 532, 511, 503, 513, 498, 527, 489, 500, 506] payload_rows_per_dst=[11072, 10986, 10978, 10810, 10950, 10898, 11004, 10910] dup_rows_per_dst=[5512, 5318, 5624, 5432, 5366, 5250, 5376, 5586] combine_rows_per_src=[11072, 10986, 10978, 10810, 10950, 10898, 11004, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626671616 dispatch_meta_bytes=458576 combine_payload_bytes=626671616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627130192 ingress_bytes=702169088 link_bytes=702169088 token_bytes=1253801808 weight_movement_bytes=75497472 total_nvlink_bytes=1329299280 weight_to_token_ratio=0.060215 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904492414e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627408896 dispatch_meta_bytes=458960 combine_payload_bytes=627408896 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627867856 ingress_bytes=853901312 link_bytes=853901312 token_bytes=1255276752 weight_movement_bytes=226492416 total_nvlink_bytes=1481769168 weight_to_token_ratio=0.180432 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082867059e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626376704 dispatch_meta_bytes=458792 combine_payload_bytes=626376704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626835496 ingress_bytes=701874176 link_bytes=701874176 token_bytes=1253212200 weight_movement_bytes=75497472 total_nvlink_bytes=1328709672 weight_to_token_ratio=0.060243 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900752514e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76380 dup_rows=43590 dup_rows_remote=37914 dedup_ratio=0.332565 router_tokens_per_expert=[486, 473, 495, 497, 481, 497, 532, 534, 526, 537, 486, 488, 544, 521, 495, 549, 479, 527, 520, 514, 515, 495, 498, 489, 532, 503, 520, 508, 504, 532, 520, 498, 539, 506, 516, 520, 483, 501, 508, 503, 479, 525, 503, 571, 517, 521, 511, 528, 480, 511, 546, 514, 582, 518, 532, 511, 499, 530, 572, 530, 515, 491, 554, 504, 471, 506, 503, 495, 530, 525, 535, 497, 506, 488, 483, 511, 543, 539, 555, 492, 538, 500, 517, 500, 472, 569, 489, 527, 503, 490, 513, 513, 477, 466, 475, 472, 510, 533, 510, 523, 487, 503, 532, 542, 501, 503, 507, 476, 514, 512, 501, 525, 527, 524, 488, 521, 522, 521, 513, 551, 529, 513, 514, 504, 481, 533, 491, 510] payload_rows_per_dst=[10856, 10804, 10844, 11102, 10986, 10854, 11028, 11008] dup_rows_per_dst=[5426, 5504, 5618, 5676, 5372, 5188, 5330, 5476] combine_rows_per_src=[10856, 10804, 10844, 11102, 10986, 10854, 11028, 11008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=629473280 dispatch_meta_bytes=458272 combine_payload_bytes=629473280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629931552 ingress_bytes=704970752 link_bytes=704970752 token_bytes=1259404832 weight_movement_bytes=75497472 total_nvlink_bytes=1334902304 weight_to_token_ratio=0.059947 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.940021457e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76668 dup_rows=43514 dup_rows_remote=37894 dedup_ratio=0.331985 router_tokens_per_expert=[518, 513, 523, 508, 509, 532, 532, 524, 476, 507, 493, 491, 511, 527, 520, 515, 522, 476, 537, 526, 521, 503, 520, 495, 515, 499, 492, 536, 469, 507, 531, 488, 513, 535, 527, 507, 476, 502, 478, 515, 505, 485, 484, 496, 537, 512, 496, 507, 522, 484, 542, 507, 527, 544, 493, 517, 483, 485, 480, 517, 521, 502, 501, 500, 532, 504, 515, 502, 524, 505, 544, 513, 491, 520, 509, 493, 529, 522, 524, 528, 510, 510, 495, 574, 494, 481, 500, 544, 499, 500, 500, 527, 523, 559, 499, 502, 543, 580, 512, 514, 545, 524, 509, 501, 516, 490, 503, 505, 525, 469, 535, 533, 545, 519, 497, 505, 509, 516, 484, 551, 509, 531, 521, 520, 497, 509, 492, 519] payload_rows_per_dst=[10878, 10864, 10908, 11014, 10890, 10982, 11042, 10980] dup_rows_per_dst=[5520, 5410, 5242, 5236, 5620, 5452, 5566, 5468] combine_rows_per_src=[10878, 10864, 10908, 11014, 10890, 10982, 11042, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87382 payload_rows_remote=76198 dup_rows=43690 dup_rows_remote=37736 dedup_ratio=0.333328 router_tokens_per_expert=[532, 555, 511, 553, 502, 472, 538, 512, 537, 501, 508, 491, 564, 496, 521, 517, 519, 479, 521, 492, 467, 513, 470, 507, 546, 512, 517, 531, 513, 521, 520, 518, 524, 545, 510, 493, 548, 553, 525, 502, 531, 518, 534, 516, 513, 528, 511, 488, 544, 531, 498, 520, 522, 527, 513, 486, 506, 531, 516, 500, 511, 497, 505, 533, 525, 475, 491, 474, 505, 486, 532, 490, 533, 494, 519, 531, 484, 520, 553, 513, 536, 502, 496, 513, 464, 572, 474, 494, 521, 486, 491, 499, 540, 507, 469, 527, 527, 505, 550, 495, 516, 507, 511, 502, 502, 482, 519, 530, 494, 518, 511, 472, 523, 538, 489, 544, 521, 506, 487, 476, 520, 518, 514, 501, 509, 478, 502, 518] payload_rows_per_dst=[11184, 11052, 10886, 10814, 10790, 10820, 10922, 10914] dup_rows_per_dst=[5954, 5624, 5444, 5248, 5324, 5362, 5360, 5374] combine_rows_per_src=[11184, 11052, 10886, 10814, 10790, 10820, 10922, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628408320 dispatch_meta_bytes=459696 combine_payload_bytes=628408320 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628868016 ingress_bytes=703905792 link_bytes=703905792 token_bytes=1257276336 weight_movement_bytes=75497472 total_nvlink_bytes=1332773808 weight_to_token_ratio=0.060048 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926516265e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625704960 dispatch_meta_bytes=457176 combine_payload_bytes=625704960 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626162136 ingress_bytes=852197376 link_bytes=852197376 token_bytes=1251867096 weight_movement_bytes=226492416 total_nvlink_bytes=1478359512 weight_to_token_ratio=0.180924 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080706229e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628064256 dispatch_meta_bytes=458248 combine_payload_bytes=628064256 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628522504 ingress_bytes=779059200 link_bytes=779059200 token_bytes=1256586760 weight_movement_bytes=150994944 total_nvlink_bytes=1407581704 weight_to_token_ratio=0.120163 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.879567265e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=624214016 dispatch_meta_bytes=455736 combine_payload_bytes=624214016 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624669752 ingress_bytes=775208960 link_bytes=775208960 token_bytes=1248883768 weight_movement_bytes=150994944 total_nvlink_bytes=1399878712 weight_to_token_ratio=0.120904 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.830740802e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87852 payload_rows_remote=76712 dup_rows=43220 dup_rows_remote=37648 dedup_ratio=0.329742 router_tokens_per_expert=[492, 519, 547, 533, 523, 484, 519, 481, 506, 481, 539, 524, 503, 533, 540, 550, 520, 524, 518, 485, 522, 553, 505, 503, 563, 497, 529, 503, 538, 527, 540, 529, 509, 491, 525, 531, 486, 491, 545, 518, 530, 510, 508, 505, 514, 499, 538, 489, 518, 507, 484, 517, 523, 514, 487, 478, 490, 478, 484, 534, 479, 493, 555, 531, 518, 547, 495, 485, 517, 522, 489, 513, 537, 490, 516, 517, 520, 552, 518, 509, 516, 494, 490, 507, 514, 498, 514, 466, 520, 544, 545, 492, 498, 497, 534, 488, 505, 490, 494, 483, 509, 520, 455, 528, 514, 504, 520, 532, 530, 518, 543, 469, 507, 525, 535, 511, 542, 493, 511, 527, 477, 519, 538, 500, 500, 513, 504, 467] payload_rows_per_dst=[11002, 11140, 10940, 10900, 11010, 10942, 11010, 10908] dup_rows_per_dst=[5546, 5572, 5438, 5244, 5480, 5292, 5218, 5430] combine_rows_per_src=[11002, 11140, 10940, 10900, 11010, 10942, 11010, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87632 payload_rows_remote=76620 dup_rows=43440 dup_rows_remote=37956 dedup_ratio=0.331421 router_tokens_per_expert=[494, 500, 526, 539, 517, 547, 467, 542, 534, 524, 515, 542, 530, 514, 520, 522, 530, 508, 510, 495, 497, 481, 507, 499, 537, 529, 542, 530, 531, 547, 514, 536, 522, 504, 495, 492, 505, 510, 479, 575, 510, 535, 510, 550, 543, 505, 509, 504, 526, 496, 536, 513, 521, 531, 470, 474, 497, 491, 525, 495, 513, 498, 528, 483, 493, 480, 535, 512, 505, 476, 516, 515, 509, 483, 544, 473, 546, 505, 507, 525, 524, 519, 555, 525, 497, 524, 492, 479, 480, 507, 507, 536, 486, 503, 510, 518, 524, 496, 542, 508, 512, 488, 463, 502, 532, 485, 510, 511, 500, 507, 483, 519, 504, 513, 496, 492, 490, 512, 523, 514, 542, 517, 527, 532, 509, 509, 509, 508] payload_rows_per_dst=[10954, 11010, 11012, 10940, 10954, 10946, 10906, 10910] dup_rows_per_dst=[5712, 5576, 5484, 5254, 5294, 5378, 5258, 5484] combine_rows_per_src=[10954, 11010, 11012, 10940, 10954, 10946, 10906, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87346 payload_rows_remote=76392 dup_rows=43726 dup_rows_remote=38324 dedup_ratio=0.333603 router_tokens_per_expert=[540, 545, 562, 524, 510, 535, 474, 492, 501, 533, 491, 510, 530, 491, 468, 510, 498, 507, 548, 484, 440, 510, 499, 500, 550, 505, 511, 519, 522, 591, 529, 506, 512, 530, 526, 559, 544, 517, 480, 526, 538, 506, 498, 538, 511, 501, 501, 496, 535, 509, 534, 503, 504, 486, 512, 461, 501, 554, 528, 486, 509, 480, 526, 498, 529, 456, 533, 484, 485, 462, 472, 533, 477, 545, 503, 541, 520, 468, 528, 552, 514, 533, 495, 530, 553, 513, 547, 504, 523, 537, 463, 535, 504, 491, 495, 487, 468, 528, 565, 486, 537, 497, 492, 511, 522, 496, 514, 492, 534, 521, 498, 517, 535, 513, 554, 507, 512, 526, 499, 503, 511, 515, 513, 506, 485, 511, 542, 470] payload_rows_per_dst=[10916, 10910, 11046, 10888, 10828, 10948, 10954, 10856] dup_rows_per_dst=[5516, 5528, 5520, 5364, 5348, 5500, 5402, 5548] combine_rows_per_src=[10916, 10910, 11046, 10888, 10828, 10948, 10954, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87912 payload_rows_remote=76722 dup_rows=43160 dup_rows_remote=37530 dedup_ratio=0.329285 router_tokens_per_expert=[530, 519, 488, 515, 493, 488, 514, 519, 492, 498, 501, 492, 522, 515, 515, 509, 465, 495, 462, 476, 496, 538, 536, 511, 529, 525, 493, 488, 517, 499, 489, 494, 518, 523, 552, 492, 527, 484, 525, 531, 504, 547, 469, 529, 514, 508, 536, 519, 510, 521, 509, 500, 485, 523, 529, 501, 522, 473, 559, 522, 510, 504, 553, 500, 510, 525, 510, 510, 493, 467, 485, 512, 483, 498, 500, 497, 526, 505, 550, 538, 544, 547, 524, 520, 503, 522, 488, 506, 457, 538, 492, 539, 534, 551, 527, 467, 499, 470, 502, 546, 560, 480, 500, 542, 523, 518, 520, 522, 543, 505, 496, 507, 515, 554, 500, 521, 527, 578, 527, 520, 490, 495, 488, 523, 499, 523, 523, 530] payload_rows_per_dst=[10654, 10708, 10814, 11240, 11014, 11072, 11220, 11190] dup_rows_per_dst=[4802, 5014, 5092, 5852, 5458, 5750, 5562, 5630] combine_rows_per_src=[10654, 10708, 10814, 11240, 11014, 11072, 11220, 11190] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87346 payload_rows_remote=76510 dup_rows=43726 dup_rows_remote=38156 dedup_ratio=0.333603 router_tokens_per_expert=[502, 526, 497, 497, 498, 502, 539, 493, 517, 540, 497, 525, 509, 462, 535, 494, 508, 504, 525, 528, 492, 497, 516, 538, 520, 541, 516, 507, 491, 483, 524, 479, 510, 517, 507, 491, 531, 524, 515, 533, 478, 527, 497, 529, 560, 507, 518, 525, 507, 523, 498, 497, 548, 516, 529, 530, 464, 517, 514, 498, 509, 476, 497, 482, 534, 532, 539, 550, 514, 495, 505, 502, 502, 513, 535, 513, 513, 571, 507, 494, 451, 515, 530, 531, 496, 486, 507, 503, 498, 515, 526, 563, 572, 490, 512, 508, 528, 506, 537, 538, 507, 503, 470, 503, 477, 497, 527, 496, 491, 490, 534, 551, 494, 517, 509, 498, 511, 550, 471, 496, 549, 502, 509, 490, 541, 506, 524, 516] payload_rows_per_dst=[10860, 10950, 10978, 10862, 11054, 10836, 10916, 10890] dup_rows_per_dst=[5406, 5388, 5560, 5348, 5584, 5570, 5394, 5476] combine_rows_per_src=[10860, 10950, 10978, 10862, 11054, 10836, 10916, 10890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628424704 dispatch_meta_bytes=457440 combine_payload_bytes=628424704 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628882144 ingress_bytes=854917120 link_bytes=854917120 token_bytes=1257306848 weight_movement_bytes=226492416 total_nvlink_bytes=1483799264 weight_to_token_ratio=0.180141 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084155247e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625803264 dispatch_meta_bytes=458864 combine_payload_bytes=625803264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626262128 ingress_bytes=701300736 link_bytes=701300736 token_bytes=1252065392 weight_movement_bytes=75497472 total_nvlink_bytes=1327562864 weight_to_token_ratio=0.060298 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893480488e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627671040 dispatch_meta_bytes=458304 combine_payload_bytes=627671040 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628129344 ingress_bytes=778665984 link_bytes=778665984 token_bytes=1255800384 weight_movement_bytes=150994944 total_nvlink_bytes=1406795328 weight_to_token_ratio=0.120238 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874580732e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628506624 dispatch_meta_bytes=457008 combine_payload_bytes=628506624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628963632 ingress_bytes=704004096 link_bytes=704004096 token_bytes=1257470256 weight_movement_bytes=75497472 total_nvlink_bytes=1332967728 weight_to_token_ratio=0.060039 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927762898e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626769920 dispatch_meta_bytes=458664 combine_payload_bytes=626769920 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627228584 ingress_bytes=702267392 link_bytes=702267392 token_bytes=1253998504 weight_movement_bytes=75497472 total_nvlink_bytes=1329495976 weight_to_token_ratio=0.060205 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905739047e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87778 payload_rows_remote=76828 dup_rows=43294 dup_rows_remote=38066 dedup_ratio=0.330307 router_tokens_per_expert=[520, 520, 534, 480, 458, 510, 462, 512, 512, 533, 574, 518, 520, 543, 518, 535, 492, 533, 481, 565, 541, 515, 497, 519, 506, 517, 497, 516, 499, 516, 518, 516, 547, 533, 498, 496, 522, 526, 504, 516, 528, 552, 538, 537, 542, 526, 517, 520, 486, 502, 495, 529, 537, 504, 498, 509, 535, 483, 524, 515, 495, 477, 487, 513, 518, 503, 492, 531, 456, 508, 520, 494, 478, 500, 520, 528, 497, 509, 510, 524, 534, 532, 532, 507, 513, 516, 453, 484, 512, 489, 464, 509, 494, 534, 521, 537, 506, 522, 522, 529, 543, 527, 468, 527, 522, 493, 551, 467, 521, 497, 511, 508, 520, 493, 518, 543, 518, 499, 489, 554, 517, 514, 520, 480, 481, 492, 489, 508] payload_rows_per_dst=[10986, 10992, 11142, 10950, 10814, 10922, 11020, 10952] dup_rows_per_dst=[5512, 5464, 5662, 5228, 5362, 5340, 5408, 5318] combine_rows_per_src=[10986, 10992, 11142, 10950, 10814, 10922, 11020, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87172 payload_rows_remote=76220 dup_rows=43900 dup_rows_remote=38338 dedup_ratio=0.334930 router_tokens_per_expert=[508, 523, 556, 540, 522, 520, 498, 478, 515, 540, 497, 520, 530, 512, 502, 516, 513, 483, 537, 528, 510, 532, 509, 511, 548, 523, 516, 542, 518, 506, 483, 506, 492, 522, 495, 498, 509, 528, 478, 500, 508, 490, 506, 520, 497, 516, 493, 534, 517, 496, 521, 495, 532, 540, 521, 500, 478, 505, 505, 501, 527, 512, 507, 534, 541, 539, 528, 493, 508, 506, 529, 494, 512, 506, 533, 514, 521, 487, 534, 512, 501, 507, 484, 502, 550, 485, 530, 556, 521, 485, 466, 521, 492, 484, 507, 482, 517, 526, 480, 530, 539, 539, 520, 530, 528, 493, 503, 503, 512, 484, 497, 520, 540, 463, 555, 494, 496, 513, 491, 522, 498, 555, 499, 498, 518, 503, 515, 506] payload_rows_per_dst=[11036, 10986, 10728, 10810, 10952, 10904, 10900, 10856] dup_rows_per_dst=[5518, 5544, 5444, 5572, 5562, 5242, 5542, 5476] combine_rows_per_src=[11036, 10986, 10728, 10810, 10952, 10904, 10900, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87132 payload_rows_remote=76074 dup_rows=43940 dup_rows_remote=38346 dedup_ratio=0.335236 router_tokens_per_expert=[531, 508, 497, 515, 511, 520, 523, 548, 524, 492, 514, 530, 551, 518, 501, 485, 521, 511, 451, 502, 515, 542, 478, 536, 494, 532, 496, 508, 515, 520, 538, 518, 509, 498, 518, 473, 492, 515, 510, 504, 514, 534, 498, 508, 540, 499, 510, 510, 508, 525, 517, 515, 512, 528, 517, 532, 527, 531, 546, 537, 518, 495, 492, 510, 499, 549, 482, 535, 522, 502, 481, 536, 551, 475, 513, 519, 498, 505, 533, 479, 554, 533, 549, 540, 481, 494, 508, 520, 513, 482, 514, 493, 511, 507, 495, 521, 514, 496, 463, 515, 512, 516, 518, 512, 514, 507, 550, 512, 540, 505, 481, 526, 472, 514, 515, 498, 503, 518, 511, 483, 478, 559, 506, 491, 525, 495, 511, 495] payload_rows_per_dst=[11058, 10788, 10846, 10952, 10862, 10882, 10950, 10794] dup_rows_per_dst=[5594, 5502, 5366, 5668, 5496, 5548, 5412, 5354] combine_rows_per_src=[11058, 10788, 10846, 10952, 10862, 10882, 10950, 10794] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87804 payload_rows_remote=76750 dup_rows=43268 dup_rows_remote=37816 dedup_ratio=0.330109 router_tokens_per_expert=[524, 515, 490, 519, 495, 521, 519, 486, 474, 574, 503, 510, 493, 536, 565, 488, 543, 499, 538, 531, 544, 535, 500, 521, 491, 504, 502, 526, 542, 491, 474, 512, 495, 496, 494, 474, 458, 503, 480, 509, 509, 490, 503, 473, 535, 525, 498, 520, 497, 486, 532, 520, 486, 516, 517, 510, 559, 499, 482, 510, 498, 519, 527, 542, 510, 545, 514, 513, 524, 543, 491, 499, 497, 520, 484, 539, 524, 503, 516, 488, 487, 528, 553, 526, 536, 481, 573, 499, 496, 477, 520, 528, 500, 470, 561, 563, 534, 521, 471, 533, 496, 504, 534, 541, 499, 505, 572, 478, 507, 563, 505, 514, 505, 507, 504, 512, 507, 530, 493, 525, 503, 512, 494, 488, 542, 508, 531, 463] payload_rows_per_dst=[10994, 11054, 10818, 11014, 10960, 11074, 11014, 10876] dup_rows_per_dst=[5430, 5452, 5106, 5386, 5460, 5522, 5540, 5372] combine_rows_per_src=[10994, 11054, 10818, 11014, 10960, 11074, 11014, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=629374976 dispatch_meta_bytes=459576 combine_payload_bytes=629374976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629834552 ingress_bytes=704872448 link_bytes=704872448 token_bytes=1259209528 weight_movement_bytes=75497472 total_nvlink_bytes=1334707000 weight_to_token_ratio=0.059956 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.938774824e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=624394240 dispatch_meta_bytes=458232 combine_payload_bytes=624394240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624852472 ingress_bytes=699891712 link_bytes=699891712 token_bytes=1249246712 weight_movement_bytes=75497472 total_nvlink_bytes=1324744184 weight_to_token_ratio=0.060434 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.875612080e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=623198208 dispatch_meta_bytes=457680 combine_payload_bytes=623198208 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=623655888 ingress_bytes=925188096 link_bytes=925188096 token_bytes=1246854096 weight_movement_bytes=301989888 total_nvlink_bytes=1548843984 weight_to_token_ratio=0.242201 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.173268736e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76630 dup_rows=43514 dup_rows_remote=37960 dedup_ratio=0.331985 router_tokens_per_expert=[512, 468, 528, 521, 502, 499, 495, 497, 478, 522, 521, 498, 496, 498, 509, 533, 489, 522, 524, 495, 500, 536, 515, 516, 493, 533, 531, 480, 503, 515, 525, 474, 537, 514, 485, 520, 520, 494, 514, 485, 518, 484, 518, 504, 517, 526, 491, 496, 492, 541, 533, 496, 513, 499, 567, 531, 509, 507, 535, 524, 494, 524, 472, 537, 534, 490, 515, 495, 531, 482, 514, 536, 515, 477, 468, 516, 522, 482, 508, 540, 489, 537, 520, 519, 550, 537, 528, 517, 545, 481, 484, 516, 514, 543, 490, 498, 500, 499, 534, 468, 512, 524, 546, 492, 506, 518, 516, 521, 533, 514, 516, 542, 538, 478, 521, 536, 537, 504, 516, 488, 538, 519, 517, 517, 515, 502, 496, 555] payload_rows_per_dst=[10870, 10922, 10940, 10988, 10916, 10996, 10928, 10998] dup_rows_per_dst=[5284, 5380, 5306, 5560, 5334, 5540, 5554, 5556] combine_rows_per_src=[10870, 10922, 10940, 10988, 10916, 10996, 10928, 10998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87576 payload_rows_remote=76572 dup_rows=43496 dup_rows_remote=38216 dedup_ratio=0.331848 router_tokens_per_expert=[520, 532, 509, 500, 505, 502, 492, 499, 520, 519, 522, 506, 518, 499, 520, 490, 507, 504, 495, 517, 511, 516, 557, 484, 487, 526, 532, 523, 507, 528, 490, 529, 477, 501, 510, 510, 527, 514, 491, 517, 528, 519, 547, 486, 511, 496, 500, 501, 512, 471, 501, 527, 539, 478, 514, 520, 522, 514, 527, 485, 530, 561, 507, 477, 551, 487, 513, 526, 485, 531, 530, 527, 502, 540, 508, 512, 518, 490, 511, 515, 521, 539, 494, 506, 509, 498, 513, 538, 503, 472, 540, 546, 508, 515, 529, 516, 576, 501, 495, 536, 531, 533, 480, 530, 518, 535, 529, 533, 533, 464, 481, 482, 480, 523, 511, 524, 493, 496, 497, 497, 490, 520, 527, 506, 534, 522, 494, 486] payload_rows_per_dst=[10832, 10906, 10898, 10662, 11230, 11016, 11028, 11004] dup_rows_per_dst=[5474, 5520, 5372, 4980, 5990, 5382, 5498, 5280] combine_rows_per_src=[10832, 10906, 10898, 10662, 11230, 11016, 11028, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87428 payload_rows_remote=76456 dup_rows=43644 dup_rows_remote=38206 dedup_ratio=0.332977 router_tokens_per_expert=[512, 482, 473, 517, 479, 492, 538, 476, 471, 512, 496, 537, 514, 533, 482, 496, 526, 521, 535, 462, 507, 515, 497, 529, 470, 497, 513, 490, 504, 491, 547, 517, 502, 503, 525, 466, 495, 521, 489, 551, 520, 498, 529, 506, 514, 517, 520, 549, 533, 487, 518, 515, 552, 514, 495, 515, 525, 499, 532, 520, 522, 538, 562, 539, 530, 533, 474, 494, 521, 539, 520, 497, 521, 486, 491, 505, 544, 487, 511, 489, 510, 566, 556, 527, 531, 483, 500, 520, 518, 525, 547, 496, 508, 520, 527, 522, 510, 491, 549, 519, 507, 455, 510, 513, 513, 525, 508, 536, 487, 556, 522, 561, 513, 485, 517, 496, 522, 487, 513, 514, 522, 516, 504, 461, 513, 506, 522, 483] payload_rows_per_dst=[10764, 10868, 10972, 11064, 10830, 11100, 11008, 10822] dup_rows_per_dst=[5256, 5374, 5438, 5668, 5454, 5612, 5516, 5326] combine_rows_per_src=[10764, 10868, 10972, 11064, 10830, 11100, 11008, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87358 payload_rows_remote=76562 dup_rows=43714 dup_rows_remote=38348 dedup_ratio=0.333511 router_tokens_per_expert=[492, 540, 481, 500, 514, 482, 509, 562, 512, 539, 493, 519, 506, 507, 539, 512, 551, 523, 512, 494, 503, 518, 498, 493, 502, 471, 518, 497, 518, 483, 526, 479, 501, 529, 488, 498, 478, 493, 532, 531, 499, 513, 511, 487, 495, 495, 507, 566, 553, 541, 470, 547, 543, 519, 502, 538, 518, 510, 500, 514, 500, 528, 527, 496, 514, 494, 510, 490, 526, 511, 498, 532, 480, 534, 507, 498, 502, 552, 508, 487, 516, 493, 525, 484, 485, 519, 483, 511, 502, 480, 508, 502, 549, 503, 538, 483, 489, 549, 549, 538, 561, 539, 488, 507, 499, 515, 496, 494, 519, 552, 498, 516, 508, 567, 494, 506, 504, 459, 527, 502, 530, 528, 538, 513, 507, 530, 528, 540] payload_rows_per_dst=[10908, 10852, 10912, 11066, 10912, 10796, 10970, 10942] dup_rows_per_dst=[5506, 5320, 5334, 5546, 5374, 5366, 5648, 5620] combine_rows_per_src=[10908, 10852, 10912, 11066, 10912, 10796, 10970, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628736000 dispatch_meta_bytes=458264 combine_payload_bytes=628736000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629194264 ingress_bytes=704233472 link_bytes=704233472 token_bytes=1257930264 weight_movement_bytes=75497472 total_nvlink_bytes=1333427736 weight_to_token_ratio=0.060017 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930671709e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627752960 dispatch_meta_bytes=458360 combine_payload_bytes=627752960 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=628211320 ingress_bytes=1005240320 link_bytes=1005240320 token_bytes=1255964280 weight_movement_bytes=377487360 total_nvlink_bytes=1633451640 weight_to_token_ratio=0.300556 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.274786224e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627277824 dispatch_meta_bytes=459152 combine_payload_bytes=627277824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627736976 ingress_bytes=702775296 link_bytes=702775296 token_bytes=1255014800 weight_movement_bytes=75497472 total_nvlink_bytes=1330512272 weight_to_token_ratio=0.060157 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912179984e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626327552 dispatch_meta_bytes=458648 combine_payload_bytes=626327552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626786200 ingress_bytes=701825024 link_bytes=701825024 token_bytes=1253113752 weight_movement_bytes=75497472 total_nvlink_bytes=1328611224 weight_to_token_ratio=0.060248 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900129198e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=459640 combine_payload_bytes=627195904 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627655544 ingress_bytes=778190848 link_bytes=778190848 token_bytes=1254851448 weight_movement_bytes=150994944 total_nvlink_bytes=1405846392 weight_to_token_ratio=0.120329 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.868555339e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87594 payload_rows_remote=76638 dup_rows=43478 dup_rows_remote=38006 dedup_ratio=0.331711 router_tokens_per_expert=[477, 495, 505, 515, 487, 490, 539, 543, 548, 534, 487, 503, 500, 500, 512, 513, 506, 495, 552, 500, 495, 491, 525, 500, 527, 521, 548, 537, 522, 510, 488, 530, 556, 504, 523, 532, 537, 539, 530, 549, 537, 509, 512, 491, 531, 499, 503, 538, 539, 520, 507, 462, 517, 512, 482, 512, 528, 522, 488, 502, 494, 521, 504, 474, 503, 489, 516, 527, 507, 505, 519, 507, 552, 504, 533, 513, 519, 519, 504, 497, 501, 504, 548, 517, 483, 503, 491, 514, 507, 509, 520, 454, 516, 519, 503, 515, 514, 530, 542, 501, 515, 515, 532, 514, 501, 510, 538, 563, 489, 469, 506, 535, 508, 489, 517, 502, 525, 458, 518, 540, 520, 499, 492, 545, 459, 449, 537, 517] payload_rows_per_dst=[10954, 11026, 11020, 10904, 10956, 10878, 11072, 10784] dup_rows_per_dst=[5342, 5468, 5760, 5264, 5472, 5330, 5476, 5366] combine_rows_per_src=[10954, 11026, 11020, 10904, 10956, 10878, 11072, 10784] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87268 payload_rows_remote=76404 dup_rows=43804 dup_rows_remote=38330 dedup_ratio=0.334198 router_tokens_per_expert=[509, 559, 520, 494, 499, 490, 497, 503, 499, 514, 512, 465, 525, 524, 534, 523, 520, 526, 501, 512, 537, 503, 502, 464, 484, 506, 532, 523, 539, 514, 496, 513, 556, 523, 507, 517, 540, 510, 536, 504, 499, 559, 505, 504, 515, 485, 538, 505, 515, 510, 497, 538, 491, 478, 505, 534, 505, 535, 507, 516, 519, 530, 521, 468, 485, 530, 485, 500, 507, 510, 535, 524, 509, 486, 487, 474, 496, 534, 491, 507, 514, 460, 493, 492, 497, 484, 518, 527, 554, 510, 535, 525, 511, 507, 543, 509, 520, 479, 546, 494, 542, 521, 517, 524, 517, 530, 537, 472, 565, 534, 515, 508, 519, 539, 509, 488, 478, 519, 480, 511, 541, 523, 493, 523, 539, 525, 487, 491] payload_rows_per_dst=[10888, 10942, 11024, 10864, 10802, 10902, 10918, 10928] dup_rows_per_dst=[5446, 5402, 5582, 5474, 5318, 5456, 5724, 5402] combine_rows_per_src=[10888, 10942, 11024, 10864, 10802, 10902, 10918, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87352 payload_rows_remote=76106 dup_rows=43720 dup_rows_remote=37842 dedup_ratio=0.333557 router_tokens_per_expert=[503, 550, 515, 460, 511, 503, 533, 513, 518, 520, 521, 458, 537, 504, 503, 520, 528, 540, 514, 545, 478, 495, 496, 516, 535, 537, 508, 492, 532, 543, 478, 527, 485, 517, 503, 526, 494, 519, 524, 505, 496, 507, 524, 530, 528, 501, 509, 508, 520, 507, 504, 545, 537, 503, 494, 539, 486, 504, 546, 522, 472, 540, 507, 517, 512, 501, 503, 488, 493, 515, 522, 470, 482, 520, 519, 460, 510, 508, 537, 492, 494, 526, 528, 502, 528, 472, 507, 499, 500, 518, 515, 505, 487, 494, 476, 500, 496, 547, 468, 519, 517, 521, 482, 515, 545, 530, 499, 509, 539, 506, 501, 509, 500, 515, 546, 518, 534, 516, 521, 505, 514, 554, 497, 538, 530, 538, 524, 548] payload_rows_per_dst=[11246, 11092, 11036, 11018, 10762, 10786, 10446, 10966] dup_rows_per_dst=[5878, 5494, 5554, 5614, 5302, 5316, 4732, 5830] combine_rows_per_src=[11246, 11092, 11036, 11018, 10762, 10786, 10446, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627818496 dispatch_meta_bytes=458576 combine_payload_bytes=627818496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628277072 ingress_bytes=703315968 link_bytes=703315968 token_bytes=1256095568 weight_movement_bytes=75497472 total_nvlink_bytes=1331593040 weight_to_token_ratio=0.060105 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919036466e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87382 payload_rows_remote=76524 dup_rows=43690 dup_rows_remote=38188 dedup_ratio=0.333328 router_tokens_per_expert=[476, 511, 511, 520, 490, 528, 495, 543, 499, 497, 524, 516, 507, 507, 555, 493, 485, 515, 535, 528, 529, 518, 520, 484, 476, 522, 519, 481, 542, 494, 531, 501, 539, 520, 498, 530, 548, 533, 514, 526, 528, 499, 501, 471, 482, 530, 473, 506, 519, 530, 494, 513, 510, 544, 526, 501, 501, 517, 522, 501, 555, 499, 510, 525, 547, 534, 566, 532, 525, 501, 520, 496, 560, 497, 495, 507, 575, 527, 526, 522, 506, 501, 510, 515, 496, 533, 499, 495, 472, 508, 547, 510, 471, 508, 530, 520, 496, 525, 488, 524, 501, 479, 483, 494, 529, 486, 492, 516, 524, 544, 477, 507, 526, 465, 527, 514, 512, 532, 524, 464, 555, 511, 502, 469, 507, 505, 480, 510] payload_rows_per_dst=[10890, 10858, 10926, 10982, 11112, 10926, 10812, 10876] dup_rows_per_dst=[5454, 5502, 5470, 5552, 5748, 5316, 5318, 5330] combine_rows_per_src=[10890, 10858, 10926, 10982, 11112, 10926, 10812, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=623460352 dispatch_meta_bytes=455792 combine_payload_bytes=623460352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623916144 ingress_bytes=698957824 link_bytes=698957824 token_bytes=1247376496 weight_movement_bytes=75497472 total_nvlink_bytes=1322873968 weight_to_token_ratio=0.060525 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.863769066e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625901568 dispatch_meta_bytes=458936 combine_payload_bytes=625901568 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626360504 ingress_bytes=776896512 link_bytes=776896512 token_bytes=1252262072 weight_movement_bytes=150994944 total_nvlink_bytes=1403257016 weight_to_token_ratio=0.120578 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.852141337e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76628 dup_rows=43584 dup_rows_remote=38292 dedup_ratio=0.332520 router_tokens_per_expert=[488, 521, 572, 521, 504, 538, 485, 537, 512, 533, 474, 501, 539, 529, 488, 517, 543, 541, 522, 511, 491, 476, 486, 505, 516, 492, 527, 546, 521, 521, 473, 498, 510, 520, 516, 532, 499, 503, 493, 531, 493, 512, 496, 503, 564, 491, 510, 557, 524, 512, 532, 520, 504, 528, 555, 542, 521, 548, 518, 514, 514, 516, 518, 509, 477, 491, 514, 518, 473, 520, 521, 520, 501, 504, 496, 514, 540, 526, 472, 502, 523, 513, 521, 466, 517, 504, 506, 521, 530, 499, 518, 484, 531, 521, 492, 504, 502, 514, 508, 506, 508, 519, 457, 489, 479, 514, 499, 501, 506, 506, 498, 570, 534, 502, 527, 507, 513, 517, 452, 503, 521, 490, 513, 539, 517, 535, 509, 509] payload_rows_per_dst=[11032, 10974, 11064, 11108, 10770, 10816, 10860, 10864] dup_rows_per_dst=[5486, 5364, 5396, 5642, 5408, 5484, 5292, 5512] combine_rows_per_src=[11032, 10974, 11064, 11108, 10770, 10816, 10860, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87594 payload_rows_remote=76500 dup_rows=43478 dup_rows_remote=37962 dedup_ratio=0.331711 router_tokens_per_expert=[514, 499, 482, 507, 511, 510, 495, 527, 516, 502, 482, 514, 476, 492, 521, 501, 565, 509, 479, 512, 473, 517, 502, 475, 553, 490, 530, 515, 531, 507, 540, 518, 490, 489, 505, 515, 521, 529, 505, 486, 476, 504, 523, 517, 488, 541, 559, 502, 526, 527, 564, 469, 504, 512, 545, 500, 510, 549, 505, 526, 478, 527, 521, 515, 480, 550, 511, 494, 476, 517, 493, 489, 484, 522, 509, 533, 537, 545, 519, 505, 492, 530, 551, 515, 507, 491, 501, 506, 531, 528, 542, 537, 522, 471, 516, 544, 471, 524, 482, 505, 493, 546, 552, 511, 543, 507, 514, 512, 510, 513, 495, 525, 536, 513, 498, 517, 494, 523, 520, 535, 499, 497, 485, 528, 543, 501, 521, 482] payload_rows_per_dst=[10866, 10958, 10756, 10762, 10896, 10950, 11312, 11094] dup_rows_per_dst=[5232, 5474, 5336, 5008, 5478, 5392, 6042, 5516] combine_rows_per_src=[10866, 10958, 10756, 10762, 10896, 10950, 11312, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87654 payload_rows_remote=76616 dup_rows=43418 dup_rows_remote=37952 dedup_ratio=0.331253 router_tokens_per_expert=[500, 515, 510, 553, 467, 514, 501, 509, 504, 534, 462, 507, 530, 521, 508, 522, 532, 514, 514, 500, 524, 511, 512, 486, 536, 482, 525, 503, 511, 525, 556, 518, 487, 487, 525, 519, 522, 522, 532, 523, 516, 518, 528, 504, 559, 509, 473, 528, 528, 521, 551, 494, 530, 532, 521, 515, 537, 513, 509, 486, 535, 516, 504, 511, 531, 518, 511, 526, 465, 512, 513, 489, 508, 484, 475, 534, 511, 503, 527, 519, 543, 507, 528, 504, 481, 525, 519, 513, 521, 545, 486, 504, 533, 482, 495, 479, 532, 493, 477, 500, 518, 510, 502, 489, 524, 475, 507, 526, 506, 540, 516, 485, 519, 483, 560, 501, 543, 486, 489, 490, 536, 513, 492, 525, 497, 487, 529, 534] payload_rows_per_dst=[10994, 10936, 11038, 11146, 10868, 10892, 10878, 10902] dup_rows_per_dst=[5320, 5562, 5466, 5460, 5384, 5438, 5322, 5466] combine_rows_per_src=[10994, 10936, 11038, 11146, 10868, 10892, 10878, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87510 payload_rows_remote=76430 dup_rows=43562 dup_rows_remote=38070 dedup_ratio=0.332352 router_tokens_per_expert=[528, 505, 489, 490, 513, 477, 538, 508, 530, 517, 547, 485, 525, 505, 484, 544, 525, 505, 543, 547, 494, 491, 489, 483, 461, 509, 536, 508, 523, 526, 523, 475, 492, 515, 501, 501, 517, 512, 522, 502, 508, 524, 552, 514, 457, 528, 527, 545, 512, 535, 449, 490, 523, 516, 519, 504, 506, 517, 530, 491, 513, 441, 501, 469, 540, 534, 553, 534, 523, 487, 551, 509, 514, 516, 507, 503, 495, 491, 556, 473, 490, 541, 505, 478, 483, 495, 512, 533, 554, 500, 536, 534, 553, 545, 536, 491, 512, 493, 522, 504, 492, 509, 493, 509, 466, 534, 531, 520, 513, 528, 502, 565, 511, 532, 518, 534, 526, 507, 517, 507, 505, 488, 515, 492, 531, 536, 485, 511] payload_rows_per_dst=[10906, 10874, 10996, 10760, 11000, 11080, 10952, 10942] dup_rows_per_dst=[5464, 5402, 5438, 5272, 5572, 5492, 5434, 5488] combine_rows_per_src=[10906, 10874, 10996, 10760, 11000, 11080, 10952, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=458848 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627343456 ingress_bytes=702382080 link_bytes=702382080 token_bytes=1254228064 weight_movement_bytes=75497472 total_nvlink_bytes=1329725536 weight_to_token_ratio=0.060194 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907193452e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627736576 dispatch_meta_bytes=459680 combine_payload_bytes=627736576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628196256 ingress_bytes=703234048 link_bytes=703234048 token_bytes=1255932832 weight_movement_bytes=75497472 total_nvlink_bytes=1331430304 weight_to_token_ratio=0.060113 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917997605e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626688000 dispatch_meta_bytes=457848 combine_payload_bytes=626688000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627145848 ingress_bytes=702185472 link_bytes=702185472 token_bytes=1253833848 weight_movement_bytes=75497472 total_nvlink_bytes=1329331320 weight_to_token_ratio=0.060213 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904700186e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627638272 dispatch_meta_bytes=458272 combine_payload_bytes=627638272 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=628096544 ingress_bytes=929628160 link_bytes=929628160 token_bytes=1255734816 weight_movement_bytes=301989888 total_nvlink_bytes=1557724704 weight_to_token_ratio=0.240489 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178899362e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626114560 dispatch_meta_bytes=458000 combine_payload_bytes=626114560 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626572560 ingress_bytes=852606976 link_bytes=852606976 token_bytes=1252687120 weight_movement_bytes=226492416 total_nvlink_bytes=1479179536 weight_to_token_ratio=0.180805 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.049632 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081225659e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87492 payload_rows_remote=76552 dup_rows=43580 dup_rows_remote=37952 dedup_ratio=0.332489 router_tokens_per_expert=[519, 516, 501, 518, 570, 526, 502, 522, 503, 495, 524, 496, 527, 519, 495, 470, 503, 482, 479, 503, 510, 502, 535, 540, 514, 506, 519, 488, 514, 552, 501, 489, 527, 509, 542, 512, 498, 529, 532, 533, 486, 531, 530, 502, 491, 516, 527, 514, 535, 501, 536, 468, 489, 511, 505, 525, 509, 504, 529, 495, 505, 532, 500, 496, 537, 515, 513, 544, 570, 531, 549, 511, 523, 492, 520, 495, 493, 497, 506, 488, 500, 523, 530, 534, 498, 470, 513, 475, 516, 525, 437, 526, 537, 513, 544, 489, 506, 474, 533, 479, 530, 530, 490, 510, 476, 557, 542, 508, 530, 508, 486, 489, 492, 522, 513, 498, 517, 481, 492, 511, 489, 528, 545, 534, 540, 518, 523, 512] payload_rows_per_dst=[11004, 10898, 10952, 10876, 10940, 10918, 10890, 11014] dup_rows_per_dst=[5402, 5376, 5606, 5404, 5628, 5342, 5406, 5416] combine_rows_per_src=[11004, 10898, 10952, 10876, 10940, 10918, 10890, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87378 payload_rows_remote=76426 dup_rows=43694 dup_rows_remote=37962 dedup_ratio=0.333359 router_tokens_per_expert=[487, 515, 488, 537, 506, 505, 527, 544, 508, 492, 515, 510, 506, 517, 486, 491, 487, 514, 544, 498, 487, 536, 543, 556, 489, 487, 500, 474, 470, 485, 511, 498, 566, 547, 542, 523, 554, 526, 519, 522, 497, 509, 516, 486, 522, 521, 544, 525, 507, 475, 557, 510, 530, 525, 534, 514, 490, 506, 563, 521, 523, 508, 536, 575, 539, 488, 492, 502, 497, 502, 525, 508, 492, 497, 473, 461, 523, 497, 478, 501, 519, 490, 509, 540, 519, 539, 512, 529, 512, 528, 498, 499, 515, 500, 493, 526, 514, 487, 490, 525, 537, 511, 539, 487, 513, 491, 515, 513, 527, 489, 504, 496, 489, 545, 523, 507, 526, 505, 488, 538, 535, 514, 511, 518, 517, 517, 452, 504] payload_rows_per_dst=[10952, 10918, 11040, 11156, 10734, 10812, 10858, 10908] dup_rows_per_dst=[5732, 5482, 5394, 5592, 5216, 5390, 5418, 5470] combine_rows_per_src=[10952, 10918, 11040, 11156, 10734, 10812, 10858, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87254 payload_rows_remote=76370 dup_rows=43818 dup_rows_remote=38350 dedup_ratio=0.334305 router_tokens_per_expert=[526, 523, 468, 473, 527, 537, 511, 518, 491, 505, 513, 545, 516, 516, 515, 485, 521, 519, 583, 494, 511, 539, 506, 492, 506, 527, 519, 496, 527, 511, 491, 505, 536, 510, 513, 535, 507, 500, 467, 499, 469, 484, 498, 525, 522, 493, 530, 509, 518, 518, 512, 504, 531, 499, 545, 513, 511, 493, 530, 494, 519, 506, 498, 485, 533, 547, 542, 518, 537, 473, 469, 514, 475, 489, 488, 530, 511, 525, 467, 541, 505, 542, 514, 526, 582, 520, 523, 497, 519, 538, 516, 525, 530, 499, 550, 526, 502, 484, 456, 490, 532, 523, 533, 523, 525, 510, 502, 507, 521, 521, 505, 505, 520, 517, 533, 534, 485, 519, 503, 510, 508, 469, 526, 460, 491, 495, 543, 524] payload_rows_per_dst=[10894, 10958, 10868, 10884, 10806, 11114, 10786, 10944] dup_rows_per_dst=[5444, 5536, 5326, 5468, 5512, 5710, 5492, 5330] combine_rows_per_src=[10894, 10958, 10868, 10884, 10806, 11114, 10786, 10944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627113984 dispatch_meta_bytes=458016 combine_payload_bytes=627113984 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627572000 ingress_bytes=702611456 link_bytes=702611456 token_bytes=1254685984 weight_movement_bytes=75497472 total_nvlink_bytes=1330183456 weight_to_token_ratio=0.060172 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910102262e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626081792 dispatch_meta_bytes=457552 combine_payload_bytes=626081792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626539344 ingress_bytes=701579264 link_bytes=701579264 token_bytes=1252621136 weight_movement_bytes=75497472 total_nvlink_bytes=1328118608 weight_to_token_ratio=0.060272 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897012615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87720 payload_rows_remote=76778 dup_rows=43352 dup_rows_remote=37818 dedup_ratio=0.330750 router_tokens_per_expert=[496, 509, 510, 524, 500, 513, 543, 528, 500, 505, 499, 484, 493, 503, 495, 503, 520, 472, 553, 476, 523, 526, 512, 519, 551, 483, 490, 515, 547, 541, 498, 512, 559, 488, 475, 513, 473, 510, 508, 505, 471, 532, 507, 480, 533, 530, 524, 515, 489, 508, 505, 502, 568, 506, 543, 582, 537, 469, 539, 518, 512, 560, 518, 496, 517, 501, 523, 519, 497, 506, 528, 447, 483, 523, 539, 544, 524, 518, 511, 474, 549, 530, 535, 472, 546, 553, 508, 490, 509, 494, 498, 511, 426, 530, 514, 551, 513, 500, 497, 538, 500, 497, 504, 513, 525, 510, 516, 466, 515, 503, 520, 526, 524, 505, 494, 509, 520, 563, 518, 462, 540, 480, 524, 527, 528, 462, 501, 548] payload_rows_per_dst=[10864, 10942, 10962, 11046, 10880, 11026, 10966, 11034] dup_rows_per_dst=[5346, 5534, 5284, 5658, 5428, 5406, 5320, 5376] combine_rows_per_src=[10864, 10942, 10962, 11046, 10880, 11026, 10966, 11034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625623040 dispatch_meta_bytes=458880 combine_payload_bytes=625623040 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626081920 ingress_bytes=776617984 link_bytes=776617984 token_bytes=1251704960 weight_movement_bytes=150994944 total_nvlink_bytes=1402699904 weight_to_token_ratio=0.120631 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.848609210e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87316 payload_rows_remote=76390 dup_rows=43756 dup_rows_remote=38358 dedup_ratio=0.333832 router_tokens_per_expert=[533, 501, 522, 543, 531, 530, 541, 527, 515, 501, 497, 495, 525, 517, 532, 529, 553, 526, 514, 477, 504, 545, 538, 524, 511, 538, 522, 509, 537, 489, 484, 539, 504, 494, 508, 545, 556, 529, 513, 516, 512, 505, 512, 504, 561, 505, 556, 474, 526, 488, 527, 515, 518, 508, 472, 538, 520, 499, 532, 480, 480, 503, 519, 480, 503, 510, 525, 494, 475, 501, 522, 491, 509, 503, 519, 499, 482, 503, 508, 495, 490, 480, 543, 522, 526, 508, 503, 498, 542, 507, 522, 537, 496, 509, 520, 534, 505, 527, 524, 516, 527, 542, 475, 520, 494, 505, 509, 523, 490, 485, 517, 503, 506, 529, 498, 506, 493, 508, 488, 503, 479, 514, 525, 492, 513, 512, 527, 457] payload_rows_per_dst=[10976, 11034, 10928, 10854, 10814, 10936, 10926, 10848] dup_rows_per_dst=[5702, 5586, 5660, 5356, 5264, 5538, 5398, 5252] combine_rows_per_src=[10976, 11034, 10928, 10854, 10814, 10936, 10926, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87370 payload_rows_remote=76348 dup_rows=43702 dup_rows_remote=37980 dedup_ratio=0.333420 router_tokens_per_expert=[530, 546, 464, 484, 550, 460, 509, 482, 535, 530, 504, 490, 513, 508, 464, 474, 513, 524, 550, 486, 491, 490, 525, 474, 532, 503, 533, 555, 539, 502, 499, 512, 536, 532, 534, 530, 494, 491, 519, 527, 518, 537, 522, 495, 510, 474, 505, 493, 508, 470, 530, 504, 542, 453, 543, 493, 519, 510, 469, 522, 542, 544, 523, 487, 464, 501, 538, 442, 527, 495, 519, 549, 477, 519, 493, 533, 499, 529, 537, 520, 509, 513, 514, 524, 507, 511, 512, 509, 517, 513, 521, 464, 551, 548, 536, 518, 480, 524, 511, 518, 455, 500, 524, 470, 479, 541, 506, 526, 511, 497, 565, 514, 531, 521, 533, 488, 543, 512, 538, 520, 506, 503, 515, 536, 490, 536, 542, 545] payload_rows_per_dst=[10798, 10884, 10824, 10896, 10920, 10934, 11092, 11022] dup_rows_per_dst=[5288, 5572, 5558, 5422, 5364, 5392, 5384, 5722] combine_rows_per_src=[10798, 10884, 10824, 10896, 10920, 10934, 11092, 11022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76496 dup_rows=43570 dup_rows_remote=38110 dedup_ratio=0.332413 router_tokens_per_expert=[513, 511, 501, 514, 509, 502, 502, 531, 494, 507, 503, 494, 488, 533, 493, 472, 507, 539, 496, 529, 503, 522, 514, 500, 505, 492, 516, 497, 496, 512, 507, 489, 546, 519, 492, 495, 509, 493, 536, 516, 500, 514, 522, 557, 489, 518, 512, 515, 483, 509, 510, 559, 507, 535, 542, 522, 530, 511, 525, 477, 510, 502, 537, 505, 482, 521, 499, 527, 478, 520, 493, 515, 518, 511, 528, 486, 525, 545, 551, 488, 541, 511, 474, 476, 530, 527, 504, 478, 515, 523, 519, 491, 519, 566, 492, 500, 538, 522, 494, 479, 490, 486, 531, 524, 556, 530, 543, 494, 525, 513, 543, 512, 477, 492, 504, 501, 510, 554, 548, 516, 505, 528, 489, 520, 523, 553, 524, 471] payload_rows_per_dst=[10892, 10940, 11006, 10966, 10858, 10896, 11018, 10926] dup_rows_per_dst=[5242, 5308, 5460, 5562, 5516, 5436, 5542, 5504] combine_rows_per_src=[10892, 10940, 11006, 10966, 10858, 10896, 11018, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87062 payload_rows_remote=76200 dup_rows=44010 dup_rows_remote=38392 dedup_ratio=0.335770 router_tokens_per_expert=[495, 518, 498, 557, 547, 486, 541, 552, 495, 506, 526, 562, 504, 495, 496, 507, 524, 536, 547, 511, 496, 519, 512, 486, 508, 516, 464, 511, 511, 528, 488, 500, 531, 494, 528, 512, 527, 517, 462, 518, 479, 532, 516, 532, 538, 502, 504, 532, 498, 472, 479, 472, 535, 516, 512, 503, 525, 485, 520, 527, 525, 518, 458, 487, 485, 516, 541, 505, 487, 502, 552, 497, 473, 547, 539, 560, 498, 504, 504, 509, 520, 505, 523, 539, 512, 532, 490, 515, 525, 515, 500, 524, 498, 471, 507, 564, 502, 505, 505, 511, 529, 559, 503, 522, 507, 503, 546, 534, 511, 498, 502, 508, 531, 516, 482, 529, 514, 515, 535, 508, 492, 496, 491, 472, 537, 512, 517, 487] payload_rows_per_dst=[11036, 10774, 10826, 10778, 10904, 10862, 11000, 10882] dup_rows_per_dst=[5534, 5540, 5622, 5286, 5534, 5618, 5490, 5386] combine_rows_per_src=[11036, 10774, 10826, 10778, 10904, 10862, 11000, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628965376 dispatch_meta_bytes=458384 combine_payload_bytes=628965376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629423760 ingress_bytes=704462848 link_bytes=704462848 token_bytes=1258389136 weight_movement_bytes=75497472 total_nvlink_bytes=1333886608 weight_to_token_ratio=0.059995 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.933580519e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625786880 dispatch_meta_bytes=458992 combine_payload_bytes=625786880 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626245872 ingress_bytes=776781824 link_bytes=776781824 token_bytes=1252032752 weight_movement_bytes=150994944 total_nvlink_bytes=1403027696 weight_to_token_ratio=0.120600 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850686931e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625442816 dispatch_meta_bytes=457312 combine_payload_bytes=625442816 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625900128 ingress_bytes=776437760 link_bytes=776437760 token_bytes=1251342944 weight_movement_bytes=150994944 total_nvlink_bytes=1402337888 weight_to_token_ratio=0.120666 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.846323716e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87224 payload_rows_remote=76358 dup_rows=43848 dup_rows_remote=38364 dedup_ratio=0.334534 router_tokens_per_expert=[524, 490, 494, 522, 549, 509, 525, 512, 507, 524, 527, 541, 481, 508, 567, 540, 542, 499, 524, 509, 495, 525, 507, 532, 501, 530, 491, 492, 542, 528, 500, 499, 514, 492, 557, 491, 520, 502, 496, 486, 532, 496, 521, 462, 506, 511, 533, 510, 504, 539, 526, 495, 489, 521, 527, 507, 526, 489, 490, 521, 527, 522, 518, 497, 491, 457, 517, 506, 528, 483, 530, 541, 522, 528, 489, 465, 550, 543, 525, 500, 526, 530, 527, 530, 506, 511, 466, 470, 531, 519, 454, 471, 507, 505, 518, 535, 525, 534, 532, 543, 512, 501, 492, 477, 506, 544, 532, 525, 505, 547, 512, 498, 511, 510, 497, 511, 493, 511, 532, 514, 493, 497, 547, 506, 499, 515, 473, 498] payload_rows_per_dst=[10948, 10872, 11044, 10852, 10866, 10852, 10922, 10868] dup_rows_per_dst=[5692, 5560, 5214, 5544, 5484, 5360, 5648, 5346] combine_rows_per_src=[10948, 10872, 11044, 10852, 10866, 10852, 10922, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626655232 dispatch_meta_bytes=458424 combine_payload_bytes=626655232 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627113656 ingress_bytes=853147648 link_bytes=853147648 token_bytes=1253768888 weight_movement_bytes=226492416 total_nvlink_bytes=1480261304 weight_to_token_ratio=0.180649 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081911307e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=624230400 dispatch_meta_bytes=458368 combine_payload_bytes=624230400 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=624688768 ingress_bytes=850722816 link_bytes=850722816 token_bytes=1248919168 weight_movement_bytes=226492416 total_nvlink_bytes=1475411584 weight_to_token_ratio=0.181351 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078836279e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87476 payload_rows_remote=76400 dup_rows=43596 dup_rows_remote=38124 dedup_ratio=0.332611 router_tokens_per_expert=[514, 472, 461, 501, 536, 562, 515, 537, 559, 509, 494, 492, 522, 500, 504, 518, 511, 547, 498, 493, 519, 526, 520, 507, 504, 504, 503, 543, 505, 503, 492, 501, 504, 500, 509, 492, 522, 511, 483, 495, 516, 530, 516, 506, 510, 493, 510, 501, 524, 497, 492, 554, 524, 506, 512, 453, 487, 483, 515, 517, 507, 518, 510, 526, 499, 514, 522, 535, 531, 490, 509, 505, 503, 483, 537, 543, 522, 520, 519, 493, 494, 548, 517, 509, 517, 545, 496, 473, 470, 517, 502, 498, 522, 539, 497, 538, 527, 509, 528, 509, 510, 496, 491, 546, 522, 534, 521, 518, 508, 526, 485, 500, 517, 530, 532, 510, 565, 492, 548, 488, 490, 534, 495, 502, 520, 511, 543, 527] payload_rows_per_dst=[11076, 10864, 10914, 10924, 11176, 10788, 10698, 11036] dup_rows_per_dst=[5472, 5498, 5272, 5458, 5880, 5444, 5156, 5416] combine_rows_per_src=[11076, 10864, 10914, 10924, 11176, 10788, 10698, 11036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87704 payload_rows_remote=76698 dup_rows=43368 dup_rows_remote=37904 dedup_ratio=0.330872 router_tokens_per_expert=[546, 487, 498, 508, 470, 485, 472, 524, 528, 498, 533, 505, 529, 514, 491, 515, 543, 481, 501, 476, 481, 487, 502, 570, 519, 511, 559, 477, 496, 491, 505, 483, 535, 512, 517, 528, 507, 506, 499, 556, 491, 487, 524, 496, 508, 544, 509, 530, 530, 509, 518, 501, 521, 540, 492, 471, 558, 533, 510, 493, 512, 519, 495, 533, 504, 559, 505, 497, 516, 528, 461, 504, 496, 460, 502, 488, 494, 548, 508, 522, 485, 535, 525, 512, 537, 539, 527, 526, 499, 515, 530, 503, 534, 524, 482, 519, 483, 516, 543, 484, 495, 501, 495, 547, 533, 492, 528, 509, 552, 547, 513, 518, 485, 538, 483, 526, 528, 470, 498, 542, 480, 498, 524, 551, 530, 511, 539, 524] payload_rows_per_dst=[10906, 10814, 11050, 11006, 10850, 11002, 11058, 11018] dup_rows_per_dst=[5300, 5350, 5448, 5464, 5334, 5582, 5454, 5436] combine_rows_per_src=[10906, 10814, 11050, 11006, 10850, 11002, 11058, 11018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625524736 dispatch_meta_bytes=458888 combine_payload_bytes=625524736 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625983624 ingress_bytes=701022208 link_bytes=701022208 token_bytes=1251508360 weight_movement_bytes=75497472 total_nvlink_bytes=1327005832 weight_to_token_ratio=0.060325 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889948361e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625868800 dispatch_meta_bytes=458096 combine_payload_bytes=625868800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626326896 ingress_bytes=701366272 link_bytes=701366272 token_bytes=1252195696 weight_movement_bytes=75497472 total_nvlink_bytes=1327693168 weight_to_token_ratio=0.060292 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894311577e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87374 payload_rows_remote=76474 dup_rows=43698 dup_rows_remote=38322 dedup_ratio=0.333389 router_tokens_per_expert=[530, 500, 509, 499, 532, 522, 522, 532, 501, 553, 528, 533, 498, 508, 513, 496, 477, 533, 515, 522, 496, 536, 508, 503, 503, 524, 520, 480, 528, 512, 467, 514, 534, 501, 501, 523, 496, 502, 521, 521, 526, 536, 523, 541, 519, 521, 543, 519, 475, 532, 540, 492, 465, 485, 471, 525, 503, 497, 527, 461, 547, 503, 515, 524, 517, 444, 491, 566, 516, 530, 496, 535, 547, 495, 510, 468, 532, 524, 523, 482, 500, 533, 473, 493, 564, 517, 460, 519, 508, 531, 517, 498, 492, 506, 513, 509, 515, 509, 494, 476, 489, 523, 504, 518, 536, 513, 505, 524, 524, 511, 487, 491, 545, 521, 520, 521, 515, 483, 533, 551, 514, 505, 514, 491, 538, 539, 518, 497] payload_rows_per_dst=[10958, 10900, 11096, 10840, 10914, 10854, 10840, 10972] dup_rows_per_dst=[5594, 5376, 5558, 5284, 5438, 5412, 5398, 5638] combine_rows_per_src=[10958, 10900, 11096, 10840, 10914, 10854, 10840, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87816 payload_rows_remote=76864 dup_rows=43256 dup_rows_remote=37874 dedup_ratio=0.330017 router_tokens_per_expert=[506, 497, 474, 532, 513, 564, 511, 491, 511, 499, 522, 502, 482, 509, 507, 500, 524, 547, 523, 512, 532, 531, 484, 493, 493, 528, 544, 551, 541, 477, 523, 504, 521, 513, 505, 503, 524, 503, 482, 542, 576, 500, 515, 495, 512, 505, 535, 503, 545, 557, 493, 536, 525, 504, 521, 493, 492, 497, 516, 486, 553, 548, 502, 500, 523, 513, 464, 510, 479, 505, 511, 494, 534, 516, 496, 511, 515, 502, 499, 535, 477, 520, 503, 481, 504, 528, 539, 515, 505, 522, 538, 513, 538, 547, 524, 527, 518, 472, 550, 499, 485, 519, 503, 490, 539, 522, 500, 488, 524, 512, 545, 501, 495, 506, 485, 502, 546, 505, 508, 492, 538, 489, 482, 529, 492, 503, 475, 505] payload_rows_per_dst=[10978, 11032, 10912, 11036, 10974, 11078, 10952, 10854] dup_rows_per_dst=[5262, 5582, 5556, 5500, 5240, 5484, 5382, 5250] combine_rows_per_src=[10978, 11032, 10912, 11036, 10974, 11078, 10952, 10854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628310016 dispatch_meta_bytes=458408 combine_payload_bytes=628310016 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628768424 ingress_bytes=703807488 link_bytes=703807488 token_bytes=1257078440 weight_movement_bytes=75497472 total_nvlink_bytes=1332575912 weight_to_token_ratio=0.060058 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925269632e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87352 payload_rows_remote=76346 dup_rows=43720 dup_rows_remote=38150 dedup_ratio=0.333557 router_tokens_per_expert=[545, 515, 499, 509, 477, 533, 509, 513, 522, 520, 518, 449, 496, 501, 544, 536, 484, 532, 529, 541, 563, 551, 495, 480, 485, 464, 568, 524, 503, 532, 543, 510, 514, 536, 509, 534, 536, 505, 504, 526, 529, 490, 506, 499, 489, 540, 524, 524, 515, 508, 521, 527, 542, 493, 556, 510, 505, 544, 550, 506, 539, 529, 530, 515, 532, 491, 504, 508, 504, 495, 494, 515, 500, 494, 505, 503, 482, 492, 513, 521, 507, 495, 542, 514, 505, 518, 518, 512, 516, 529, 529, 486, 504, 506, 510, 491, 521, 452, 535, 496, 507, 493, 505, 523, 501, 545, 513, 485, 524, 472, 510, 517, 514, 507, 500, 499, 505, 529, 459, 510, 493, 508, 517, 495, 488, 572, 473, 488] payload_rows_per_dst=[10896, 10782, 10740, 11148, 10618, 11096, 11066, 11006] dup_rows_per_dst=[5476, 5240, 5148, 5638, 5042, 5854, 5752, 5570] combine_rows_per_src=[10896, 10782, 10740, 11148, 10618, 11096, 11066, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87412 payload_rows_remote=76504 dup_rows=43660 dup_rows_remote=38226 dedup_ratio=0.333099 router_tokens_per_expert=[502, 503, 484, 496, 471, 480, 472, 517, 484, 540, 515, 526, 520, 522, 506, 451, 524, 482, 531, 520, 491, 478, 513, 505, 515, 504, 502, 506, 524, 519, 573, 533, 543, 488, 488, 533, 463, 492, 509, 517, 520, 514, 542, 482, 514, 534, 529, 503, 518, 521, 522, 482, 513, 520, 508, 505, 496, 480, 555, 510, 538, 521, 537, 477, 491, 512, 542, 553, 520, 476, 469, 553, 517, 507, 531, 497, 548, 537, 532, 539, 549, 503, 513, 536, 557, 498, 506, 509, 518, 565, 507, 512, 467, 548, 511, 521, 487, 510, 490, 510, 540, 508, 526, 500, 561, 489, 532, 479, 500, 511, 515, 505, 527, 466, 515, 497, 462, 508, 530, 501, 516, 513, 544, 451, 499, 532, 511, 574] payload_rows_per_dst=[10698, 11028, 10908, 10962, 11028, 11020, 10894, 10874] dup_rows_per_dst=[5280, 5412, 5434, 5444, 5620, 5620, 5432, 5418] combine_rows_per_src=[10698, 11028, 10908, 10962, 11028, 11020, 10894, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87322 payload_rows_remote=76540 dup_rows=43750 dup_rows_remote=38380 dedup_ratio=0.333786 router_tokens_per_expert=[490, 488, 510, 513, 528, 502, 525, 506, 503, 522, 530, 487, 507, 481, 528, 526, 515, 522, 512, 558, 541, 548, 477, 492, 506, 506, 525, 493, 515, 536, 501, 461, 513, 489, 505, 575, 513, 526, 544, 533, 501, 504, 466, 558, 529, 510, 541, 507, 546, 516, 501, 502, 504, 519, 518, 516, 481, 485, 517, 512, 476, 487, 510, 527, 501, 539, 564, 501, 508, 525, 535, 483, 523, 561, 484, 521, 504, 536, 506, 522, 501, 472, 512, 512, 493, 516, 515, 506, 524, 517, 523, 459, 516, 513, 498, 499, 521, 457, 492, 543, 526, 508, 486, 519, 527, 517, 522, 528, 498, 532, 508, 516, 519, 489, 508, 473, 491, 498, 480, 513, 548, 531, 510, 514, 554, 522, 526, 486] payload_rows_per_dst=[10888, 10926, 10972, 10828, 11046, 10782, 10900, 10980] dup_rows_per_dst=[5404, 5490, 5656, 5406, 5580, 5370, 5500, 5344] combine_rows_per_src=[10888, 10926, 10972, 10828, 11046, 10782, 10900, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626475008 dispatch_meta_bytes=459184 combine_payload_bytes=626475008 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626934192 ingress_bytes=777469952 link_bytes=777469952 token_bytes=1253409200 weight_movement_bytes=150994944 total_nvlink_bytes=1404404144 weight_to_token_ratio=0.120467 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.859413363e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=629669888 dispatch_meta_bytes=458952 combine_payload_bytes=629669888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630128840 ingress_bytes=705167360 link_bytes=705167360 token_bytes=1259798728 weight_movement_bytes=75497472 total_nvlink_bytes=1335296200 weight_to_token_ratio=0.059928 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.942514723e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625426432 dispatch_meta_bytes=457984 combine_payload_bytes=625426432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625884416 ingress_bytes=700923904 link_bytes=700923904 token_bytes=1251310848 weight_movement_bytes=75497472 total_nvlink_bytes=1326808320 weight_to_token_ratio=0.060335 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888701728e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87062 payload_rows_remote=76162 dup_rows=44010 dup_rows_remote=38388 dedup_ratio=0.335770 router_tokens_per_expert=[497, 499, 502, 478, 554, 505, 512, 547, 499, 538, 505, 515, 510, 508, 517, 530, 534, 508, 558, 499, 527, 527, 499, 540, 529, 473, 472, 543, 534, 471, 518, 553, 509, 508, 517, 511, 510, 488, 514, 509, 548, 485, 524, 498, 517, 503, 484, 486, 523, 521, 559, 527, 557, 504, 531, 551, 494, 506, 534, 474, 531, 464, 547, 545, 532, 500, 506, 480, 526, 481, 530, 525, 518, 523, 555, 520, 512, 513, 507, 533, 517, 495, 470, 484, 514, 512, 501, 516, 507, 525, 516, 529, 500, 534, 459, 523, 511, 489, 471, 495, 527, 494, 514, 491, 525, 475, 496, 513, 487, 472, 496, 537, 514, 536, 502, 517, 508, 495, 474, 495, 508, 544, 548, 508, 462, 541, 525, 523] payload_rows_per_dst=[10888, 11072, 10824, 11024, 10900, 10796, 10738, 10820] dup_rows_per_dst=[5544, 5498, 5398, 5712, 5622, 5408, 5248, 5580] combine_rows_per_src=[10888, 11072, 10824, 11024, 10900, 10796, 10738, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=458920 combine_payload_bytes=626720768 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627179688 ingress_bytes=853213184 link_bytes=853213184 token_bytes=1253900456 weight_movement_bytes=226492416 total_nvlink_bytes=1480392872 weight_to_token_ratio=0.180630 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081994416e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87386 payload_rows_remote=76350 dup_rows=43686 dup_rows_remote=37948 dedup_ratio=0.333298 router_tokens_per_expert=[488, 540, 461, 510, 533, 514, 504, 487, 497, 500, 472, 519, 507, 468, 562, 477, 500, 502, 515, 537, 545, 510, 523, 533, 559, 495, 493, 497, 530, 472, 495, 504, 513, 519, 498, 497, 528, 500, 485, 559, 512, 492, 509, 530, 488, 509, 545, 515, 475, 504, 511, 507, 522, 507, 556, 498, 534, 485, 528, 538, 539, 527, 526, 528, 482, 527, 506, 560, 498, 509, 516, 519, 498, 525, 545, 530, 496, 546, 515, 480, 546, 486, 484, 520, 494, 486, 529, 470, 511, 521, 464, 477, 529, 496, 507, 508, 526, 487, 511, 513, 526, 529, 514, 501, 503, 526, 542, 483, 562, 508, 477, 567, 576, 537, 508, 485, 548, 498, 512, 520, 500, 500, 525, 502, 466, 501, 528, 542] payload_rows_per_dst=[11036, 11072, 10952, 10728, 10976, 10788, 10904, 10930] dup_rows_per_dst=[5738, 5452, 5446, 5042, 5528, 5268, 5646, 5566] combine_rows_per_src=[11036, 11072, 10952, 10728, 10976, 10788, 10904, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=459680 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627475360 ingress_bytes=702513152 link_bytes=702513152 token_bytes=1254491040 weight_movement_bytes=75497472 total_nvlink_bytes=1329988512 weight_to_token_ratio=0.060182 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908855629e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76548 dup_rows=43558 dup_rows_remote=38044 dedup_ratio=0.332321 router_tokens_per_expert=[483, 523, 469, 537, 555, 513, 492, 554, 527, 519, 496, 514, 517, 487, 527, 503, 499, 531, 478, 465, 511, 497, 513, 484, 498, 521, 523, 530, 487, 517, 524, 527, 477, 506, 524, 475, 525, 520, 530, 507, 508, 543, 536, 512, 532, 521, 521, 499, 533, 465, 511, 549, 499, 539, 518, 544, 530, 539, 468, 512, 506, 500, 480, 547, 558, 509, 471, 518, 492, 520, 525, 494, 546, 523, 501, 518, 513, 526, 505, 554, 495, 530, 526, 477, 534, 493, 526, 531, 505, 513, 516, 479, 499, 505, 484, 508, 516, 505, 501, 501, 480, 486, 527, 504, 507, 539, 557, 558, 483, 515, 491, 540, 534, 526, 495, 532, 542, 500, 521, 507, 493, 500, 486, 493, 476, 538, 485, 507] payload_rows_per_dst=[10978, 10898, 10990, 10966, 10980, 10852, 10978, 10872] dup_rows_per_dst=[5454, 5312, 5482, 5514, 5566, 5390, 5442, 5398] combine_rows_per_src=[10978, 10898, 10990, 10966, 10980, 10852, 10978, 10872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=623919104 dispatch_meta_bytes=458200 combine_payload_bytes=623919104 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624377304 ingress_bytes=774914048 link_bytes=774914048 token_bytes=1248296408 weight_movement_bytes=150994944 total_nvlink_bytes=1399291352 weight_to_token_ratio=0.120961 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.827000903e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625459200 dispatch_meta_bytes=457192 combine_payload_bytes=625459200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625916392 ingress_bytes=700956672 link_bytes=700956672 token_bytes=1251375592 weight_movement_bytes=75497472 total_nvlink_bytes=1326873064 weight_to_token_ratio=0.060332 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889117272e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87598 payload_rows_remote=76500 dup_rows=43474 dup_rows_remote=37932 dedup_ratio=0.331680 router_tokens_per_expert=[544, 527, 527, 488, 522, 536, 556, 504, 531, 489, 514, 507, 508, 527, 485, 528, 492, 480, 523, 528, 525, 532, 508, 527, 544, 505, 532, 549, 510, 526, 509, 530, 498, 522, 474, 507, 507, 537, 537, 550, 508, 511, 507, 504, 514, 497, 538, 461, 502, 524, 488, 478, 488, 523, 483, 517, 547, 545, 504, 503, 485, 503, 496, 511, 513, 516, 465, 488, 532, 497, 484, 454, 529, 512, 512, 498, 495, 525, 507, 489, 492, 523, 481, 519, 498, 509, 474, 516, 518, 513, 512, 530, 533, 508, 557, 456, 507, 509, 509, 513, 545, 530, 500, 516, 463, 516, 519, 544, 513, 544, 567, 520, 506, 523, 493, 520, 517, 542, 511, 504, 527, 487, 520, 493, 489, 497, 521, 534] payload_rows_per_dst=[11020, 11098, 10858, 10812, 10784, 10922, 11094, 11010] dup_rows_per_dst=[5566, 5542, 5486, 5382, 5248, 5356, 5536, 5358] combine_rows_per_src=[11020, 11098, 10858, 10812, 10784, 10922, 11094, 11010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87166 payload_rows_remote=76318 dup_rows=43906 dup_rows_remote=38548 dedup_ratio=0.334976 router_tokens_per_expert=[485, 479, 494, 550, 552, 518, 535, 449, 487, 496, 485, 518, 491, 513, 537, 519, 489, 488, 532, 521, 486, 524, 505, 526, 549, 506, 521, 504, 544, 487, 518, 496, 514, 508, 493, 501, 526, 519, 538, 518, 565, 533, 493, 478, 499, 548, 506, 516, 533, 545, 513, 549, 493, 488, 528, 512, 488, 518, 527, 553, 498, 556, 507, 458, 493, 493, 484, 480, 478, 518, 538, 505, 516, 524, 529, 543, 477, 526, 530, 493, 508, 512, 561, 522, 499, 514, 539, 512, 542, 503, 521, 518, 514, 510, 515, 525, 515, 497, 504, 504, 479, 475, 522, 520, 548, 462, 517, 509, 518, 514, 495, 524, 477, 537, 515, 524, 501, 498, 532, 492, 535, 536, 483, 534, 503, 535, 480, 484] payload_rows_per_dst=[10850, 10870, 10972, 10918, 10858, 11012, 10848, 10838] dup_rows_per_dst=[5366, 5522, 5538, 5614, 5396, 5618, 5358, 5494] combine_rows_per_src=[10850, 10870, 10972, 10918, 10858, 11012, 10848, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76476 dup_rows=43506 dup_rows_remote=37902 dedup_ratio=0.331924 router_tokens_per_expert=[537, 513, 510, 526, 508, 510, 528, 513, 514, 501, 493, 495, 513, 547, 544, 517, 503, 526, 540, 466, 526, 536, 511, 541, 517, 490, 557, 491, 496, 511, 470, 514, 487, 502, 527, 526, 547, 547, 496, 524, 490, 514, 526, 493, 538, 518, 499, 488, 549, 468, 509, 567, 509, 512, 492, 528, 509, 480, 489, 495, 532, 489, 504, 453, 501, 484, 481, 474, 529, 536, 496, 515, 496, 494, 516, 481, 512, 514, 512, 526, 533, 484, 517, 536, 551, 547, 513, 501, 485, 475, 529, 488, 522, 517, 533, 489, 469, 485, 478, 524, 489, 497, 515, 511, 509, 511, 543, 508, 530, 512, 544, 555, 527, 499, 596, 461, 512, 509, 502, 526, 548, 519, 490, 524, 506, 531, 532, 516] payload_rows_per_dst=[10968, 10950, 10836, 10676, 11222, 10926, 10898, 11090] dup_rows_per_dst=[5394, 5440, 5234, 4994, 5864, 5514, 5462, 5604] combine_rows_per_src=[10968, 10950, 10836, 10676, 11222, 10926, 10898, 11090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627081216 dispatch_meta_bytes=458368 combine_payload_bytes=627081216 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627539584 ingress_bytes=853573632 link_bytes=853573632 token_bytes=1254620800 weight_movement_bytes=226492416 total_nvlink_bytes=1481113216 weight_to_token_ratio=0.180527 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082451515e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87582 payload_rows_remote=76536 dup_rows=43490 dup_rows_remote=37936 dedup_ratio=0.331802 router_tokens_per_expert=[525, 509, 526, 475, 491, 546, 520, 525, 506, 473, 511, 515, 523, 518, 495, 540, 521, 522, 492, 521, 522, 491, 510, 531, 498, 557, 517, 512, 507, 535, 535, 537, 504, 516, 536, 546, 516, 490, 535, 517, 486, 514, 556, 530, 508, 515, 495, 536, 485, 492, 500, 534, 518, 487, 509, 503, 527, 506, 492, 524, 536, 511, 568, 517, 508, 517, 533, 489, 534, 498, 522, 494, 475, 524, 531, 502, 505, 523, 488, 501, 497, 467, 504, 517, 481, 523, 473, 489, 506, 541, 541, 509, 511, 515, 459, 506, 519, 514, 496, 461, 505, 507, 507, 506, 504, 514, 529, 507, 521, 549, 514, 511, 502, 540, 516, 473, 516, 516, 515, 497, 508, 484, 529, 525, 537, 508, 495, 513] payload_rows_per_dst=[10872, 11078, 11046, 11010, 10882, 10796, 10986, 10912] dup_rows_per_dst=[5524, 5538, 5554, 5408, 5406, 5282, 5342, 5436] combine_rows_per_src=[10872, 11078, 11046, 11010, 10882, 10796, 10986, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626688000 dispatch_meta_bytes=457728 combine_payload_bytes=626688000 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627145728 ingress_bytes=777682944 link_bytes=777682944 token_bytes=1253833728 weight_movement_bytes=150994944 total_nvlink_bytes=1404828672 weight_to_token_ratio=0.120427 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.862114401e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87548 payload_rows_remote=76674 dup_rows=43524 dup_rows_remote=38236 dedup_ratio=0.332062 router_tokens_per_expert=[504, 528, 508, 549, 489, 472, 484, 484, 500, 530, 550, 485, 501, 506, 471, 501, 512, 523, 541, 537, 473, 505, 519, 494, 533, 508, 532, 521, 499, 507, 532, 502, 486, 558, 495, 501, 489, 507, 511, 514, 551, 521, 560, 536, 482, 505, 539, 518, 547, 546, 552, 543, 510, 500, 491, 523, 537, 506, 513, 512, 508, 533, 552, 522, 491, 527, 505, 517, 510, 517, 516, 476, 540, 516, 517, 493, 480, 545, 508, 501, 509, 480, 521, 522, 522, 511, 481, 505, 503, 512, 501, 492, 497, 526, 525, 474, 451, 511, 479, 536, 528, 549, 516, 503, 519, 503, 469, 513, 516, 491, 491, 494, 503, 483, 487, 522, 546, 514, 518, 505, 537, 531, 539, 493, 486, 543, 534, 518] payload_rows_per_dst=[10892, 10940, 10948, 11090, 10966, 10874, 10846, 10992] dup_rows_per_dst=[5232, 5536, 5598, 5700, 5352, 5288, 5292, 5526] combine_rows_per_src=[10892, 10940, 10948, 11090, 10966, 10874, 10846, 10992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625197056 dispatch_meta_bytes=459464 combine_payload_bytes=625197056 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=625656520 ingress_bytes=776192000 link_bytes=776192000 token_bytes=1250853576 weight_movement_bytes=150994944 total_nvlink_bytes=1401848520 weight_to_token_ratio=0.120714 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.843207133e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87752 payload_rows_remote=76758 dup_rows=43320 dup_rows_remote=38002 dedup_ratio=0.330505 router_tokens_per_expert=[518, 547, 509, 560, 492, 534, 511, 514, 479, 522, 473, 497, 528, 498, 542, 523, 550, 502, 523, 487, 516, 510, 467, 553, 479, 471, 528, 486, 534, 488, 544, 467, 501, 498, 530, 516, 509, 482, 520, 532, 492, 519, 508, 503, 527, 521, 545, 506, 549, 523, 483, 529, 505, 534, 514, 507, 513, 521, 547, 494, 537, 516, 520, 502, 504, 482, 543, 470, 510, 499, 553, 480, 544, 538, 514, 514, 514, 476, 518, 497, 517, 495, 499, 504, 520, 485, 507, 521, 532, 507, 527, 533, 523, 525, 497, 503, 511, 504, 526, 512, 492, 528, 507, 531, 503, 475, 535, 472, 522, 515, 499, 541, 493, 535, 514, 497, 501, 497, 540, 524, 517, 512, 491, 499, 498, 516, 520, 503] payload_rows_per_dst=[10976, 10874, 11016, 11036, 10994, 10946, 10942, 10968] dup_rows_per_dst=[5518, 5336, 5402, 5552, 5318, 5444, 5404, 5346] combine_rows_per_src=[10976, 10874, 11016, 11036, 10994, 10946, 10942, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626491392 dispatch_meta_bytes=457512 combine_payload_bytes=626491392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626948904 ingress_bytes=701988864 link_bytes=701988864 token_bytes=1253440296 weight_movement_bytes=75497472 total_nvlink_bytes=1328937768 weight_to_token_ratio=0.060232 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.902206920e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76578 dup_rows=43528 dup_rows_remote=38192 dedup_ratio=0.332092 router_tokens_per_expert=[528, 508, 506, 517, 484, 526, 492, 501, 490, 495, 502, 531, 515, 527, 530, 484, 518, 464, 474, 521, 498, 483, 487, 527, 521, 526, 500, 522, 503, 541, 514, 513, 488, 523, 487, 479, 521, 509, 504, 486, 559, 519, 555, 477, 498, 517, 546, 519, 533, 495, 560, 541, 481, 495, 520, 539, 504, 527, 516, 562, 505, 504, 490, 493, 503, 505, 515, 474, 524, 473, 541, 500, 540, 531, 523, 485, 516, 512, 531, 518, 512, 497, 560, 501, 530, 536, 521, 497, 511, 507, 537, 495, 527, 492, 531, 517, 492, 512, 536, 465, 498, 556, 543, 535, 520, 500, 466, 512, 514, 525, 544, 502, 515, 517, 499, 480, 486, 497, 534, 540, 509, 527, 509, 530, 526, 480, 541, 464] payload_rows_per_dst=[10966, 10902, 10886, 10994, 10938, 10964, 11002, 10892] dup_rows_per_dst=[5336, 5296, 5488, 5536, 5444, 5588, 5424, 5416] combine_rows_per_src=[10966, 10902, 10886, 10994, 10938, 10964, 11002, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626982912 dispatch_meta_bytes=457888 combine_payload_bytes=626982912 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627440800 ingress_bytes=777977856 link_bytes=777977856 token_bytes=1254423712 weight_movement_bytes=150994944 total_nvlink_bytes=1405418656 weight_to_token_ratio=0.120370 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.865854301e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628113408 dispatch_meta_bytes=459640 combine_payload_bytes=628113408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628573048 ingress_bytes=703610880 link_bytes=703610880 token_bytes=1256686456 weight_movement_bytes=75497472 total_nvlink_bytes=1332183928 weight_to_token_ratio=0.060077 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922776366e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628801536 dispatch_meta_bytes=459040 combine_payload_bytes=628801536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629260576 ingress_bytes=704299008 link_bytes=704299008 token_bytes=1258062112 weight_movement_bytes=75497472 total_nvlink_bytes=1333559584 weight_to_token_ratio=0.060011 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.931502797e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87340 payload_rows_remote=76478 dup_rows=43732 dup_rows_remote=38244 dedup_ratio=0.333649 router_tokens_per_expert=[522, 501, 549, 520, 531, 510, 538, 550, 549, 510, 511, 483, 488, 495, 529, 510, 496, 471, 532, 528, 553, 517, 498, 493, 528, 489, 514, 483, 484, 497, 502, 490, 475, 509, 481, 513, 501, 533, 570, 501, 484, 490, 524, 474, 498, 490, 512, 550, 487, 495, 527, 456, 498, 478, 525, 514, 518, 537, 497, 525, 510, 526, 509, 573, 493, 505, 529, 499, 508, 534, 508, 495, 483, 512, 529, 519, 528, 504, 512, 542, 482, 549, 490, 504, 508, 537, 490, 518, 519, 515, 544, 528, 518, 524, 506, 539, 527, 498, 526, 500, 554, 546, 545, 522, 509, 517, 494, 539, 522, 497, 508, 515, 483, 485, 480, 504, 522, 508, 535, 510, 527, 499, 516, 491, 512, 529, 483, 511] payload_rows_per_dst=[10990, 10896, 10850, 10862, 10888, 10982, 11050, 10822] dup_rows_per_dst=[5602, 5254, 5360, 5488, 5512, 5560, 5588, 5368] combine_rows_per_src=[10990, 10896, 10850, 10862, 10888, 10982, 11050, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=459080 combine_payload_bytes=627326976 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627786056 ingress_bytes=853819392 link_bytes=853819392 token_bytes=1255113032 weight_movement_bytes=226492416 total_nvlink_bytes=1481605448 weight_to_token_ratio=0.180456 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082763173e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87784 payload_rows_remote=76876 dup_rows=43288 dup_rows_remote=37898 dedup_ratio=0.330261 router_tokens_per_expert=[486, 539, 528, 509, 530, 547, 499, 510, 513, 504, 527, 469, 525, 503, 511, 552, 512, 567, 498, 501, 463, 512, 551, 500, 497, 507, 532, 491, 493, 535, 494, 496, 512, 503, 506, 497, 520, 525, 466, 500, 517, 520, 520, 538, 504, 493, 484, 488, 533, 518, 516, 495, 528, 557, 526, 498, 501, 526, 475, 500, 518, 530, 511, 537, 508, 485, 500, 514, 484, 500, 472, 510, 520, 487, 529, 523, 539, 490, 521, 516, 533, 527, 542, 524, 473, 513, 516, 512, 502, 522, 509, 498, 527, 524, 523, 514, 520, 515, 515, 510, 497, 511, 521, 504, 489, 503, 524, 533, 498, 549, 508, 552, 490, 480, 527, 534, 510, 525, 486, 499, 513, 519, 539, 492, 520, 499, 514, 520] payload_rows_per_dst=[11048, 10908, 10924, 11018, 10868, 11062, 10954, 11002] dup_rows_per_dst=[5456, 5390, 5262, 5520, 5328, 5456, 5544, 5332] combine_rows_per_src=[11048, 10908, 10924, 11018, 10868, 11062, 10954, 11002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87510 payload_rows_remote=76470 dup_rows=43562 dup_rows_remote=38126 dedup_ratio=0.332352 router_tokens_per_expert=[500, 534, 513, 533, 557, 526, 506, 517, 516, 525, 540, 485, 532, 494, 542, 499, 506, 529, 494, 517, 477, 530, 531, 532, 489, 495, 524, 513, 497, 505, 550, 488, 502, 512, 478, 489, 542, 477, 524, 508, 504, 512, 516, 554, 526, 501, 492, 484, 529, 513, 486, 539, 534, 526, 526, 488, 522, 480, 506, 500, 508, 510, 531, 521, 523, 530, 525, 505, 496, 522, 543, 511, 506, 504, 466, 509, 482, 516, 503, 526, 536, 499, 495, 512, 537, 503, 522, 467, 475, 507, 517, 538, 502, 509, 506, 506, 511, 524, 530, 550, 511, 510, 526, 480, 544, 528, 501, 519, 511, 519, 484, 490, 483, 513, 496, 486, 541, 516, 497, 509, 570, 476, 529, 512, 505, 509, 507, 515] payload_rows_per_dst=[11062, 10892, 10940, 10962, 10976, 10790, 11040, 10848] dup_rows_per_dst=[5576, 5462, 5302, 5476, 5358, 5472, 5436, 5480] combine_rows_per_src=[11062, 10892, 10940, 10962, 10976, 10790, 11040, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626507776 dispatch_meta_bytes=458888 combine_payload_bytes=626507776 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626966664 ingress_bytes=777502720 link_bytes=777502720 token_bytes=1253474440 weight_movement_bytes=150994944 total_nvlink_bytes=1404469384 weight_to_token_ratio=0.120461 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.859828907e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87432 payload_rows_remote=76458 dup_rows=43640 dup_rows_remote=38208 dedup_ratio=0.332947 router_tokens_per_expert=[522, 472, 513, 503, 495, 478, 491, 510, 526, 490, 543, 556, 508, 517, 540, 532, 501, 501, 495, 522, 526, 518, 498, 498, 502, 528, 513, 503, 514, 521, 536, 481, 514, 505, 527, 504, 515, 498, 501, 562, 531, 508, 575, 493, 539, 543, 534, 529, 542, 510, 491, 552, 522, 519, 481, 516, 560, 500, 538, 514, 554, 454, 522, 465, 506, 514, 514, 502, 547, 524, 518, 527, 489, 503, 526, 504, 492, 503, 506, 480, 505, 529, 477, 513, 490, 503, 499, 501, 521, 492, 519, 507, 525, 499, 503, 502, 530, 494, 496, 511, 536, 530, 516, 535, 506, 532, 485, 541, 530, 495, 511, 518, 481, 511, 514, 512, 492, 515, 521, 518, 452, 548, 515, 486, 480, 470, 533, 511] payload_rows_per_dst=[10652, 10946, 11246, 10906, 10886, 10836, 10986, 10974] dup_rows_per_dst=[5128, 5368, 5834, 5490, 5424, 5362, 5602, 5432] combine_rows_per_src=[10652, 10946, 11246, 10906, 10886, 10836, 10986, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629768192 dispatch_meta_bytes=459096 combine_payload_bytes=629768192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630227288 ingress_bytes=705265664 link_bytes=705265664 token_bytes=1259995480 weight_movement_bytes=75497472 total_nvlink_bytes=1335492952 weight_to_token_ratio=0.059919 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.943761356e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87452 payload_rows_remote=76444 dup_rows=43620 dup_rows_remote=38244 dedup_ratio=0.332794 router_tokens_per_expert=[481, 506, 497, 494, 524, 491, 484, 506, 544, 499, 559, 513, 487, 533, 534, 497, 521, 490, 481, 529, 499, 501, 521, 491, 505, 480, 515, 488, 492, 498, 508, 513, 518, 511, 470, 523, 545, 510, 507, 496, 559, 512, 498, 532, 489, 512, 533, 477, 488, 472, 521, 505, 490, 486, 531, 586, 525, 515, 478, 513, 525, 510, 502, 521, 503, 505, 502, 553, 478, 474, 526, 551, 555, 493, 524, 506, 509, 543, 515, 516, 539, 532, 523, 510, 509, 527, 520, 509, 534, 514, 495, 515, 540, 537, 531, 543, 516, 541, 535, 524, 481, 519, 523, 468, 480, 506, 492, 480, 520, 485, 538, 516, 498, 515, 492, 524, 500, 498, 500, 517, 504, 551, 534, 538, 501, 479, 524, 565] payload_rows_per_dst=[10894, 10774, 11008, 10950, 10994, 11034, 10824, 10974] dup_rows_per_dst=[5404, 5290, 5376, 5386, 5512, 5722, 5424, 5506] combine_rows_per_src=[10894, 10774, 11008, 10950, 10994, 11034, 10824, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626442240 dispatch_meta_bytes=458384 combine_payload_bytes=626442240 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626900624 ingress_bytes=852934656 link_bytes=852934656 token_bytes=1253342864 weight_movement_bytes=226492416 total_nvlink_bytes=1479835280 weight_to_token_ratio=0.180711 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081641203e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76604 dup_rows=43566 dup_rows_remote=38202 dedup_ratio=0.332382 router_tokens_per_expert=[522, 511, 538, 493, 481, 484, 532, 524, 525, 507, 489, 534, 492, 532, 529, 512, 526, 515, 528, 497, 508, 550, 469, 526, 492, 472, 546, 541, 503, 508, 505, 522, 512, 489, 509, 491, 534, 526, 491, 478, 509, 509, 482, 549, 512, 494, 487, 521, 458, 501, 483, 513, 536, 520, 505, 540, 516, 529, 510, 479, 502, 520, 542, 513, 505, 481, 500, 516, 511, 534, 468, 532, 546, 520, 550, 550, 505, 516, 492, 466, 540, 523, 519, 508, 504, 489, 502, 482, 490, 525, 505, 505, 512, 508, 510, 511, 546, 533, 486, 541, 505, 535, 518, 484, 533, 525, 521, 589, 524, 495, 532, 488, 497, 512, 530, 485, 501, 540, 509, 505, 502, 498, 515, 517, 534, 496, 526, 516] payload_rows_per_dst=[10856, 10962, 10856, 11000, 10918, 10902, 11094, 10918] dup_rows_per_dst=[5554, 5454, 5330, 5334, 5466, 5364, 5616, 5448] combine_rows_per_src=[10856, 10962, 10856, 11000, 10918, 10902, 11094, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76698 dup_rows=43558 dup_rows_remote=38016 dedup_ratio=0.332321 router_tokens_per_expert=[521, 545, 477, 506, 523, 503, 499, 501, 523, 554, 491, 512, 528, 552, 553, 505, 537, 544, 534, 498, 497, 492, 472, 509, 486, 480, 512, 550, 525, 455, 526, 510, 530, 506, 512, 499, 497, 483, 505, 512, 490, 493, 514, 514, 488, 540, 537, 516, 488, 482, 500, 547, 508, 518, 521, 507, 502, 455, 531, 486, 484, 461, 533, 574, 560, 487, 511, 508, 536, 515, 490, 523, 538, 542, 480, 511, 473, 488, 486, 531, 543, 555, 507, 478, 544, 516, 496, 487, 510, 528, 480, 517, 501, 522, 474, 514, 490, 488, 585, 520, 521, 508, 487, 510, 516, 560, 521, 496, 535, 496, 505, 545, 502, 525, 548, 525, 533, 507, 500, 500, 524, 509, 495, 544, 489, 507, 505, 536] payload_rows_per_dst=[10934, 10916, 10984, 10908, 10816, 10924, 11054, 10978] dup_rows_per_dst=[5652, 5338, 5288, 5286, 5542, 5420, 5512, 5520] combine_rows_per_src=[10934, 10916, 10984, 10908, 10816, 10924, 11054, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87512 payload_rows_remote=76630 dup_rows=43560 dup_rows_remote=38134 dedup_ratio=0.332336 router_tokens_per_expert=[518, 509, 546, 486, 519, 526, 482, 463, 510, 513, 488, 502, 497, 527, 514, 487, 539, 491, 487, 514, 503, 496, 493, 494, 524, 496, 539, 527, 551, 482, 532, 499, 532, 538, 512, 521, 506, 532, 528, 506, 506, 505, 498, 508, 496, 549, 540, 508, 513, 525, 523, 489, 527, 534, 541, 536, 506, 512, 514, 529, 490, 479, 513, 520, 517, 474, 496, 509, 522, 530, 473, 577, 545, 478, 494, 534, 558, 490, 478, 516, 536, 509, 503, 511, 513, 567, 535, 521, 522, 453, 486, 495, 509, 528, 481, 483, 492, 547, 494, 511, 562, 494, 489, 500, 502, 494, 528, 544, 520, 546, 529, 524, 509, 491, 510, 532, 523, 485, 530, 472, 464, 508, 506, 521, 518, 511, 519, 528] payload_rows_per_dst=[10882, 11154, 11014, 11062, 11024, 10792, 10708, 10876] dup_rows_per_dst=[5426, 5924, 5556, 5440, 5556, 5378, 4902, 5378] combine_rows_per_src=[10882, 11154, 11014, 11062, 11024, 10792, 10708, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626343936 dispatch_meta_bytes=458664 combine_payload_bytes=626343936 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626802600 ingress_bytes=701841408 link_bytes=701841408 token_bytes=1253146536 weight_movement_bytes=75497472 total_nvlink_bytes=1328644008 weight_to_token_ratio=0.060246 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900336970e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626229248 dispatch_meta_bytes=458752 combine_payload_bytes=626229248 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626688000 ingress_bytes=701726720 link_bytes=701726720 token_bytes=1252917248 weight_movement_bytes=75497472 total_nvlink_bytes=1328414720 weight_to_token_ratio=0.060257 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898882565e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=459224 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627999192 ingress_bytes=778534912 link_bytes=778534912 token_bytes=1255539160 weight_movement_bytes=150994944 total_nvlink_bytes=1406534104 weight_to_token_ratio=0.120263 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872918555e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628310016 dispatch_meta_bytes=458856 combine_payload_bytes=628310016 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628768872 ingress_bytes=703807488 link_bytes=703807488 token_bytes=1257078888 weight_movement_bytes=75497472 total_nvlink_bytes=1332576360 weight_to_token_ratio=0.060058 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925269632e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627752960 dispatch_meta_bytes=459056 combine_payload_bytes=627752960 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628212016 ingress_bytes=854245376 link_bytes=854245376 token_bytes=1255964976 weight_movement_bytes=226492416 total_nvlink_bytes=1482457392 weight_to_token_ratio=0.180333 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083303381e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76556 dup_rows=43546 dup_rows_remote=38182 dedup_ratio=0.332230 router_tokens_per_expert=[499, 542, 519, 514, 517, 519, 459, 507, 515, 538, 506, 506, 549, 547, 543, 494, 481, 509, 517, 523, 557, 500, 509, 535, 497, 509, 504, 509, 529, 473, 506, 489, 470, 525, 490, 509, 546, 475, 490, 497, 512, 505, 528, 501, 514, 526, 503, 485, 495, 510, 546, 518, 509, 474, 519, 468, 528, 513, 518, 482, 557, 504, 494, 532, 541, 512, 451, 541, 494, 505, 533, 487, 505, 509, 495, 506, 512, 532, 541, 538, 527, 476, 522, 504, 540, 540, 449, 528, 533, 520, 524, 526, 517, 533, 504, 536, 497, 543, 503, 508, 507, 487, 506, 528, 520, 523, 525, 500, 497, 522, 547, 475, 533, 500, 558, 531, 493, 490, 519, 510, 512, 501, 500, 513, 530, 494, 489, 530] payload_rows_per_dst=[10972, 10924, 10908, 10970, 10942, 10972, 10880, 10958] dup_rows_per_dst=[5576, 5370, 5244, 5364, 5462, 5586, 5496, 5448] combine_rows_per_src=[10972, 10924, 10908, 10970, 10942, 10972, 10880, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87472 payload_rows_remote=76524 dup_rows=43600 dup_rows_remote=38306 dedup_ratio=0.332642 router_tokens_per_expert=[484, 529, 489, 504, 544, 488, 487, 501, 496, 532, 495, 501, 520, 484, 479, 512, 507, 463, 522, 496, 501, 537, 509, 482, 522, 482, 507, 512, 492, 535, 536, 518, 522, 493, 545, 526, 557, 487, 524, 488, 472, 542, 515, 523, 561, 495, 515, 501, 505, 516, 507, 464, 529, 500, 516, 499, 549, 518, 529, 519, 491, 502, 516, 525, 553, 525, 498, 519, 505, 521, 503, 525, 518, 550, 536, 508, 543, 505, 555, 446, 493, 478, 503, 508, 520, 549, 524, 503, 508, 537, 565, 529, 493, 456, 525, 536, 449, 524, 507, 520, 535, 526, 535, 460, 474, 528, 554, 515, 519, 485, 484, 496, 517, 560, 528, 489, 518, 518, 551, 514, 511, 488, 489, 574, 485, 520, 487, 522] payload_rows_per_dst=[10912, 10948, 10942, 10998, 11010, 10990, 10782, 10890] dup_rows_per_dst=[5178, 5294, 5590, 5372, 5610, 5464, 5440, 5652] combine_rows_per_src=[10912, 10948, 10942, 10998, 11010, 10990, 10782, 10890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87530 payload_rows_remote=76524 dup_rows=43542 dup_rows_remote=38050 dedup_ratio=0.332199 router_tokens_per_expert=[480, 539, 536, 533, 508, 531, 505, 526, 502, 558, 493, 496, 504, 511, 524, 511, 491, 506, 536, 533, 485, 528, 506, 509, 514, 504, 474, 479, 491, 482, 522, 530, 478, 532, 526, 515, 535, 518, 526, 508, 496, 543, 517, 510, 515, 495, 518, 508, 504, 515, 579, 529, 546, 530, 540, 514, 518, 531, 490, 502, 546, 529, 481, 523, 502, 510, 501, 518, 493, 522, 514, 515, 502, 547, 498, 458, 503, 473, 538, 529, 462, 487, 508, 516, 517, 522, 491, 518, 508, 491, 482, 535, 507, 526, 516, 506, 533, 499, 513, 509, 503, 491, 545, 520, 530, 530, 538, 518, 503, 512, 497, 508, 499, 509, 505, 518, 493, 507, 498, 503, 505, 533, 458, 539, 528, 504, 492, 517] payload_rows_per_dst=[11014, 10856, 11030, 11058, 10836, 10830, 11006, 10900] dup_rows_per_dst=[5500, 5324, 5450, 5696, 5410, 5354, 5492, 5316] combine_rows_per_src=[11014, 10856, 11030, 11058, 10836, 10830, 11006, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76384 dup_rows=43654 dup_rows_remote=37936 dedup_ratio=0.333054 router_tokens_per_expert=[504, 514, 516, 485, 506, 506, 494, 465, 498, 523, 489, 483, 502, 514, 490, 491, 505, 521, 531, 505, 533, 496, 486, 470, 541, 484, 524, 495, 489, 503, 532, 531, 504, 566, 534, 508, 549, 529, 531, 485, 473, 518, 506, 522, 491, 543, 465, 480, 573, 529, 486, 505, 561, 515, 493, 542, 500, 506, 511, 520, 538, 526, 516, 529, 497, 482, 532, 481, 490, 499, 515, 537, 532, 489, 508, 494, 497, 482, 510, 515, 540, 493, 509, 535, 506, 505, 537, 525, 542, 529, 487, 503, 478, 497, 501, 518, 506, 511, 529, 530, 509, 499, 501, 540, 463, 542, 557, 517, 508, 505, 535, 491, 506, 513, 504, 526, 527, 524, 557, 527, 546, 530, 510, 503, 523, 512, 512, 528] payload_rows_per_dst=[10736, 10810, 10880, 10916, 10996, 11076, 10970, 11034] dup_rows_per_dst=[5224, 5482, 5210, 5252, 5656, 5638, 5474, 5718] combine_rows_per_src=[10736, 10810, 10880, 10916, 10996, 11076, 10970, 11034] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627146752 dispatch_meta_bytes=458952 combine_payload_bytes=627146752 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627605704 ingress_bytes=778141696 link_bytes=778141696 token_bytes=1254752456 weight_movement_bytes=150994944 total_nvlink_bytes=1405747400 weight_to_token_ratio=0.120338 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.867932023e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=459320 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627343928 ingress_bytes=702382080 link_bytes=702382080 token_bytes=1254228536 weight_movement_bytes=75497472 total_nvlink_bytes=1329726008 weight_to_token_ratio=0.060194 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907193452e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87744 payload_rows_remote=76704 dup_rows=43328 dup_rows_remote=37776 dedup_ratio=0.330566 router_tokens_per_expert=[472, 526, 541, 525, 497, 503, 534, 483, 530, 532, 527, 524, 516, 511, 549, 498, 522, 488, 529, 528, 527, 513, 514, 507, 530, 515, 508, 465, 499, 520, 496, 513, 566, 521, 509, 519, 521, 510, 499, 525, 512, 526, 557, 500, 518, 474, 501, 538, 515, 489, 534, 540, 509, 525, 500, 504, 471, 485, 477, 522, 502, 483, 519, 528, 493, 510, 519, 502, 492, 532, 524, 500, 511, 493, 529, 474, 500, 488, 541, 485, 504, 538, 525, 520, 518, 515, 499, 478, 468, 528, 492, 466, 539, 503, 519, 518, 502, 500, 550, 555, 546, 508, 505, 515, 560, 536, 487, 483, 515, 477, 507, 505, 490, 528, 504, 516, 515, 541, 508, 550, 527, 490, 515, 491, 519, 522, 484, 521] payload_rows_per_dst=[10952, 11036, 11040, 10824, 10966, 10906, 10952, 11068] dup_rows_per_dst=[5584, 5312, 5552, 5382, 5220, 5354, 5550, 5374] combine_rows_per_src=[10952, 11036, 11040, 10824, 10966, 10906, 10952, 11068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87510 payload_rows_remote=76634 dup_rows=43562 dup_rows_remote=38162 dedup_ratio=0.332352 router_tokens_per_expert=[542, 485, 520, 527, 525, 491, 517, 501, 552, 509, 511, 494, 537, 540, 503, 499, 482, 513, 507, 540, 488, 551, 521, 528, 573, 502, 515, 492, 499, 515, 462, 520, 517, 501, 488, 516, 518, 505, 501, 548, 481, 493, 519, 524, 529, 522, 523, 512, 516, 531, 502, 497, 525, 496, 500, 501, 515, 496, 502, 500, 510, 487, 535, 528, 474, 498, 505, 532, 537, 468, 503, 513, 506, 514, 502, 536, 504, 498, 485, 516, 531, 497, 498, 483, 520, 483, 501, 536, 508, 495, 528, 517, 502, 477, 511, 551, 501, 519, 487, 483, 464, 524, 547, 533, 519, 504, 524, 529, 512, 518, 509, 548, 552, 509, 508, 501, 504, 486, 552, 522, 519, 543, 499, 549, 527, 506, 509, 501] payload_rows_per_dst=[10998, 11004, 10932, 10896, 10962, 10876, 10902, 10940] dup_rows_per_dst=[5508, 5412, 5462, 5386, 5220, 5400, 5540, 5634] combine_rows_per_src=[10998, 11004, 10932, 10896, 10962, 10876, 10902, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87342 payload_rows_remote=76364 dup_rows=43730 dup_rows_remote=38110 dedup_ratio=0.333633 router_tokens_per_expert=[521, 493, 525, 520, 505, 517, 506, 495, 508, 508, 502, 524, 520, 494, 525, 486, 522, 505, 529, 529, 525, 503, 501, 493, 544, 526, 484, 482, 506, 518, 490, 547, 488, 509, 507, 514, 483, 504, 469, 493, 466, 505, 517, 520, 554, 521, 502, 543, 525, 480, 530, 517, 540, 556, 562, 466, 512, 513, 477, 518, 508, 521, 540, 544, 558, 507, 519, 530, 529, 484, 497, 515, 511, 491, 532, 542, 533, 507, 525, 519, 517, 544, 507, 510, 488, 506, 497, 461, 510, 501, 502, 522, 528, 486, 484, 490, 490, 541, 499, 557, 554, 511, 499, 515, 489, 516, 498, 537, 538, 545, 537, 505, 522, 480, 486, 499, 520, 506, 494, 572, 483, 518, 461, 498, 521, 519, 488, 529] payload_rows_per_dst=[10944, 10936, 10802, 10984, 10978, 10768, 10986, 10944] dup_rows_per_dst=[5354, 5472, 5388, 5634, 5620, 5338, 5676, 5248] combine_rows_per_src=[10944, 10936, 10802, 10984, 10978, 10768, 10986, 10944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=458296 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627342904 ingress_bytes=928874496 link_bytes=928874496 token_bytes=1254227512 weight_movement_bytes=301989888 total_nvlink_bytes=1556217400 weight_to_token_ratio=0.240778 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177943610e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87096 payload_rows_remote=75962 dup_rows=43976 dup_rows_remote=38150 dedup_ratio=0.335510 router_tokens_per_expert=[481, 534, 522, 493, 479, 514, 539, 495, 521, 530, 524, 506, 508, 510, 530, 499, 496, 516, 516, 494, 552, 507, 515, 503, 498, 546, 515, 501, 503, 528, 504, 508, 514, 500, 553, 548, 493, 506, 473, 526, 533, 528, 503, 529, 474, 492, 516, 527, 514, 505, 504, 553, 445, 473, 509, 527, 493, 518, 489, 506, 557, 508, 517, 511, 497, 511, 482, 476, 518, 531, 541, 490, 508, 527, 519, 497, 525, 500, 517, 521, 512, 517, 527, 490, 525, 547, 514, 539, 553, 504, 496, 491, 455, 533, 512, 528, 526, 482, 500, 517, 548, 541, 514, 518, 487, 514, 530, 530, 542, 496, 511, 515, 492, 503, 517, 544, 487, 485, 523, 524, 526, 509, 489, 495, 514, 483, 504, 536] payload_rows_per_dst=[11134, 11044, 10898, 10852, 10836, 10930, 10812, 10590] dup_rows_per_dst=[5826, 5694, 5532, 5458, 5484, 5556, 5344, 5082] combine_rows_per_src=[11134, 11044, 10898, 10852, 10836, 10930, 10812, 10590] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625737728 dispatch_meta_bytes=457280 combine_payload_bytes=625737728 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626195008 ingress_bytes=776732672 link_bytes=776732672 token_bytes=1251932736 weight_movement_bytes=150994944 total_nvlink_bytes=1402927680 weight_to_token_ratio=0.120609 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850063615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628359168 dispatch_meta_bytes=457920 combine_payload_bytes=628359168 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628817088 ingress_bytes=779354112 link_bytes=779354112 token_bytes=1257176256 weight_movement_bytes=150994944 total_nvlink_bytes=1408171200 weight_to_token_ratio=0.120106 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.883307164e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=459184 combine_payload_bytes=627785728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628244912 ingress_bytes=703283200 link_bytes=703283200 token_bytes=1256030640 weight_movement_bytes=75497472 total_nvlink_bytes=1331528112 weight_to_token_ratio=0.060108 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918620922e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625573888 dispatch_meta_bytes=457896 combine_payload_bytes=625573888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626031784 ingress_bytes=701071360 link_bytes=701071360 token_bytes=1251605672 weight_movement_bytes=75497472 total_nvlink_bytes=1327103144 weight_to_token_ratio=0.060320 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890571677e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87700 payload_rows_remote=76852 dup_rows=43372 dup_rows_remote=37978 dedup_ratio=0.330902 router_tokens_per_expert=[493, 509, 551, 495, 545, 501, 504, 475, 508, 518, 489, 519, 505, 521, 551, 469, 522, 519, 526, 520, 454, 488, 511, 504, 508, 553, 537, 512, 495, 486, 499, 487, 506, 517, 543, 518, 509, 504, 475, 551, 499, 587, 521, 516, 477, 529, 493, 539, 515, 551, 503, 510, 488, 531, 532, 484, 509, 539, 500, 498, 501, 532, 533, 551, 511, 514, 528, 489, 517, 529, 525, 523, 469, 536, 521, 521, 478, 471, 492, 485, 505, 508, 497, 513, 463, 518, 495, 536, 473, 532, 492, 512, 524, 491, 517, 524, 545, 488, 497, 513, 523, 506, 527, 509, 471, 568, 540, 504, 509, 512, 553, 493, 489, 489, 528, 447, 519, 510, 556, 514, 517, 501, 545, 490, 533, 563, 508, 525] payload_rows_per_dst=[11064, 10848, 11008, 11064, 10882, 10796, 11136, 10902] dup_rows_per_dst=[5242, 5394, 5560, 5490, 5336, 5404, 5380, 5566] combine_rows_per_src=[11064, 10848, 11008, 11064, 10882, 10796, 11136, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87780 payload_rows_remote=76770 dup_rows=43292 dup_rows_remote=37938 dedup_ratio=0.330292 router_tokens_per_expert=[513, 546, 496, 513, 504, 486, 493, 490, 507, 506, 480, 510, 546, 493, 513, 508, 538, 493, 542, 493, 499, 531, 505, 517, 487, 500, 498, 475, 572, 524, 522, 528, 525, 537, 493, 535, 482, 509, 538, 496, 494, 520, 507, 503, 493, 538, 501, 511, 516, 528, 486, 517, 519, 477, 493, 516, 519, 540, 506, 513, 532, 502, 512, 506, 522, 489, 493, 548, 492, 512, 495, 530, 538, 515, 509, 480, 508, 503, 527, 473, 540, 510, 509, 550, 469, 495, 518, 547, 538, 463, 554, 554, 497, 512, 510, 530, 479, 468, 495, 547, 482, 494, 499, 495, 509, 530, 504, 518, 533, 502, 516, 502, 518, 497, 543, 546, 524, 532, 527, 545, 518, 499, 516, 489, 523, 530, 518, 516] payload_rows_per_dst=[10960, 11040, 10900, 11010, 10908, 11034, 10878, 11050] dup_rows_per_dst=[5248, 5408, 5464, 5354, 5360, 5558, 5268, 5632] combine_rows_per_src=[10960, 11040, 10900, 11010, 10908, 11034, 10878, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=622280704 dispatch_meta_bytes=456448 combine_payload_bytes=622280704 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=622737152 ingress_bytes=697778176 link_bytes=697778176 token_bytes=1245017856 weight_movement_bytes=75497472 total_nvlink_bytes=1320515328 weight_to_token_ratio=0.060640 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.848809469e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87324 payload_rows_remote=76400 dup_rows=43748 dup_rows_remote=38308 dedup_ratio=0.333771 router_tokens_per_expert=[502, 530, 523, 526, 517, 492, 469, 541, 503, 502, 520, 521, 504, 473, 512, 542, 543, 542, 497, 525, 502, 483, 528, 500, 514, 496, 517, 517, 553, 519, 493, 524, 530, 519, 506, 537, 486, 508, 562, 521, 550, 532, 484, 495, 575, 504, 523, 504, 519, 532, 468, 547, 514, 508, 505, 524, 495, 545, 475, 501, 517, 481, 478, 540, 491, 514, 518, 512, 541, 513, 547, 540, 471, 546, 541, 482, 515, 498, 553, 497, 509, 495, 490, 490, 532, 539, 491, 520, 491, 524, 537, 501, 500, 462, 475, 498, 507, 509, 506, 491, 522, 503, 491, 517, 531, 494, 517, 492, 490, 522, 540, 550, 497, 541, 500, 496, 492, 547, 531, 492, 456, 466, 497, 507, 512, 538, 513, 521] payload_rows_per_dst=[10982, 10968, 10976, 10906, 10824, 10822, 10924, 10922] dup_rows_per_dst=[5372, 5538, 5696, 5330, 5734, 5286, 5440, 5352] combine_rows_per_src=[10982, 10968, 10976, 10906, 10824, 10822, 10924, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=629571584 dispatch_meta_bytes=459320 combine_payload_bytes=629571584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630030904 ingress_bytes=705069056 link_bytes=705069056 token_bytes=1259602488 weight_movement_bytes=75497472 total_nvlink_bytes=1335099960 weight_to_token_ratio=0.059938 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.941268090e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628899840 dispatch_meta_bytes=458832 combine_payload_bytes=628899840 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629358672 ingress_bytes=779894784 link_bytes=779894784 token_bytes=1258258512 weight_movement_bytes=150994944 total_nvlink_bytes=1409253456 weight_to_token_ratio=0.120003 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.890163646e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87324 payload_rows_remote=76054 dup_rows=43748 dup_rows_remote=37750 dedup_ratio=0.333771 router_tokens_per_expert=[536, 519, 543, 492, 566, 495, 507, 499, 488, 510, 513, 496, 530, 482, 515, 509, 495, 512, 511, 522, 525, 519, 507, 512, 481, 503, 516, 531, 569, 535, 535, 501, 534, 466, 478, 553, 512, 504, 522, 487, 481, 494, 480, 509, 499, 506, 489, 546, 518, 498, 513, 513, 510, 501, 531, 495, 531, 528, 493, 539, 490, 551, 531, 501, 503, 522, 512, 490, 520, 513, 521, 518, 498, 505, 528, 521, 526, 531, 491, 523, 502, 515, 492, 535, 526, 502, 562, 514, 510, 498, 505, 503, 520, 531, 516, 511, 524, 512, 543, 450, 538, 511, 509, 532, 499, 533, 517, 549, 538, 476, 480, 501, 465, 532, 522, 524, 493, 483, 471, 523, 466, 511, 497, 503, 537, 508, 533, 515] payload_rows_per_dst=[10876, 11118, 10680, 10760, 10852, 10944, 10824, 11270] dup_rows_per_dst=[5396, 5548, 4860, 5204, 5574, 5540, 5628, 5998] combine_rows_per_src=[10876, 11118, 10680, 10760, 10852, 10944, 10824, 11270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87304 payload_rows_remote=76388 dup_rows=43768 dup_rows_remote=38168 dedup_ratio=0.333923 router_tokens_per_expert=[535, 501, 481, 498, 519, 516, 499, 531, 526, 540, 542, 512, 514, 510, 558, 483, 530, 514, 527, 485, 501, 479, 500, 514, 515, 501, 521, 498, 508, 519, 489, 497, 528, 504, 537, 519, 562, 447, 537, 517, 500, 512, 499, 497, 503, 542, 542, 512, 486, 526, 524, 491, 493, 488, 506, 492, 513, 511, 532, 539, 518, 559, 471, 505, 497, 514, 515, 506, 513, 573, 497, 499, 495, 507, 517, 509, 537, 531, 521, 513, 547, 487, 530, 491, 503, 538, 522, 501, 501, 479, 530, 539, 509, 494, 542, 508, 542, 530, 517, 511, 505, 488, 458, 488, 503, 515, 525, 512, 523, 523, 531, 498, 507, 530, 487, 495, 498, 509, 524, 521, 513, 509, 472, 515, 483, 507, 529, 528] payload_rows_per_dst=[10970, 10880, 10916, 10816, 10926, 10908, 10880, 11008] dup_rows_per_dst=[5560, 5316, 5600, 5492, 5562, 5534, 5458, 5246] combine_rows_per_src=[10970, 10880, 10916, 10816, 10926, 10908, 10880, 11008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87616 payload_rows_remote=76874 dup_rows=43456 dup_rows_remote=37996 dedup_ratio=0.331543 router_tokens_per_expert=[505, 548, 508, 508, 513, 539, 521, 510, 496, 516, 513, 513, 498, 497, 477, 491, 501, 536, 483, 494, 509, 515, 501, 484, 485, 486, 542, 521, 511, 520, 476, 495, 493, 486, 550, 569, 501, 539, 528, 489, 496, 521, 493, 498, 523, 527, 523, 521, 482, 504, 573, 527, 568, 533, 513, 517, 499, 542, 532, 539, 550, 505, 516, 511, 534, 517, 487, 539, 533, 514, 522, 510, 509, 488, 477, 485, 513, 497, 501, 475, 505, 505, 533, 472, 496, 501, 489, 491, 498, 523, 524, 526, 521, 551, 525, 522, 513, 550, 549, 539, 533, 510, 515, 485, 506, 519, 534, 503, 545, 468, 543, 501, 501, 529, 505, 491, 484, 493, 503, 497, 500, 523, 467, 499, 508, 524, 505, 531] payload_rows_per_dst=[11012, 10818, 10966, 11192, 10742, 10940, 11064, 10882] dup_rows_per_dst=[5294, 5300, 5548, 5630, 5460, 5424, 5562, 5238] combine_rows_per_src=[11012, 10818, 10966, 11192, 10742, 10940, 11064, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625868800 dispatch_meta_bytes=458832 combine_payload_bytes=625868800 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626327632 ingress_bytes=776863744 link_bytes=776863744 token_bytes=1252196432 weight_movement_bytes=150994944 total_nvlink_bytes=1403191376 weight_to_token_ratio=0.120584 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851725792e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87662 payload_rows_remote=76854 dup_rows=43410 dup_rows_remote=38076 dedup_ratio=0.331192 router_tokens_per_expert=[504, 503, 500, 542, 518, 515, 488, 516, 508, 493, 532, 514, 515, 524, 494, 478, 518, 525, 514, 536, 512, 560, 539, 513, 527, 497, 537, 537, 528, 532, 521, 543, 541, 525, 557, 522, 498, 531, 513, 532, 491, 541, 522, 493, 504, 493, 517, 491, 519, 506, 509, 521, 503, 518, 508, 532, 512, 498, 480, 464, 501, 516, 526, 512, 501, 547, 529, 491, 502, 521, 492, 498, 485, 510, 519, 489, 519, 497, 520, 535, 480, 480, 512, 526, 514, 503, 469, 473, 520, 502, 537, 529, 516, 504, 525, 481, 516, 529, 493, 521, 532, 524, 513, 498, 503, 440, 484, 518, 548, 537, 456, 511, 543, 497, 483, 519, 515, 512, 502, 497, 481, 514, 522, 524, 544, 512, 529, 514] payload_rows_per_dst=[11018, 11108, 11014, 10924, 10868, 10808, 10958, 10964] dup_rows_per_dst=[5270, 5770, 5528, 5326, 5442, 5334, 5288, 5452] combine_rows_per_src=[11018, 11108, 11014, 10924, 10868, 10808, 10958, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=623034368 dispatch_meta_bytes=455216 combine_payload_bytes=623034368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623489584 ingress_bytes=698531840 link_bytes=698531840 token_bytes=1246523952 weight_movement_bytes=75497472 total_nvlink_bytes=1322021424 weight_to_token_ratio=0.060566 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.858366989e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87452 payload_rows_remote=76528 dup_rows=43620 dup_rows_remote=38076 dedup_ratio=0.332794 router_tokens_per_expert=[496, 500, 522, 522, 500, 519, 505, 510, 499, 537, 523, 531, 505, 510, 516, 539, 541, 512, 546, 525, 490, 489, 489, 526, 496, 514, 520, 475, 462, 490, 461, 525, 498, 506, 505, 512, 491, 497, 519, 481, 514, 559, 532, 552, 511, 498, 497, 509, 488, 493, 523, 490, 508, 560, 506, 516, 527, 525, 508, 504, 514, 526, 530, 549, 495, 498, 502, 538, 522, 513, 518, 543, 527, 500, 509, 537, 493, 499, 544, 522, 493, 526, 542, 520, 552, 520, 517, 501, 498, 484, 508, 505, 497, 533, 523, 531, 535, 559, 486, 517, 540, 531, 493, 539, 496, 511, 500, 505, 531, 484, 498, 485, 494, 500, 515, 484, 553, 497, 483, 498, 502, 501, 499, 540, 453, 500, 550, 504] payload_rows_per_dst=[10924, 10776, 10920, 11042, 11008, 11122, 10760, 10900] dup_rows_per_dst=[5544, 5346, 5524, 5492, 5430, 5700, 5338, 5246] combine_rows_per_src=[10924, 10776, 10920, 11042, 11008, 11122, 10760, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625770496 dispatch_meta_bytes=458224 combine_payload_bytes=625770496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626228720 ingress_bytes=776765440 link_bytes=776765440 token_bytes=1251999216 weight_movement_bytes=150994944 total_nvlink_bytes=1402994160 weight_to_token_ratio=0.120603 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850479159e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=629751808 dispatch_meta_bytes=459480 combine_payload_bytes=629751808 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=630211288 ingress_bytes=856244224 link_bytes=856244224 token_bytes=1259963096 weight_movement_bytes=226492416 total_nvlink_bytes=1486455512 weight_to_token_ratio=0.179761 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085838202e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87356 payload_rows_remote=76474 dup_rows=43716 dup_rows_remote=38150 dedup_ratio=0.333527 router_tokens_per_expert=[554, 532, 500, 499, 488, 547, 525, 528, 556, 469, 512, 526, 486, 520, 469, 479, 537, 490, 509, 559, 523, 573, 507, 472, 517, 523, 477, 537, 480, 516, 521, 491, 524, 558, 471, 529, 497, 500, 464, 508, 467, 497, 514, 512, 516, 512, 499, 455, 537, 507, 535, 538, 544, 514, 488, 491, 544, 478, 500, 487, 511, 521, 495, 512, 503, 501, 512, 502, 483, 530, 499, 497, 517, 514, 499, 529, 535, 535, 510, 545, 480, 535, 521, 528, 496, 507, 505, 486, 478, 493, 562, 516, 528, 497, 538, 519, 522, 508, 517, 588, 502, 512, 496, 516, 526, 499, 495, 513, 536, 515, 523, 507, 506, 495, 498, 517, 553, 488, 485, 496, 522, 516, 547, 504, 492, 503, 528, 564] payload_rows_per_dst=[10932, 10882, 10806, 10886, 10958, 10946, 10970, 10976] dup_rows_per_dst=[5464, 5566, 5240, 5518, 5464, 5432, 5580, 5452] combine_rows_per_src=[10932, 10882, 10806, 10886, 10958, 10946, 10970, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=629587968 dispatch_meta_bytes=459720 combine_payload_bytes=629587968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630047688 ingress_bytes=705085440 link_bytes=705085440 token_bytes=1259635656 weight_movement_bytes=75497472 total_nvlink_bytes=1335133128 weight_to_token_ratio=0.059936 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.941475862e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87462 payload_rows_remote=76500 dup_rows=43610 dup_rows_remote=38168 dedup_ratio=0.332718 router_tokens_per_expert=[498, 518, 536, 522, 566, 502, 521, 501, 506, 512, 537, 492, 485, 512, 563, 503, 519, 519, 494, 494, 464, 506, 507, 552, 515, 539, 482, 503, 538, 493, 513, 534, 486, 492, 483, 513, 475, 516, 515, 503, 489, 521, 559, 518, 533, 535, 498, 502, 543, 476, 539, 545, 465, 504, 491, 503, 519, 516, 484, 518, 498, 535, 537, 529, 536, 478, 487, 514, 525, 535, 534, 502, 505, 487, 521, 515, 518, 550, 508, 501, 469, 504, 490, 518, 489, 550, 520, 524, 508, 547, 540, 534, 532, 511, 502, 517, 537, 530, 512, 487, 528, 508, 512, 480, 497, 506, 531, 525, 531, 515, 529, 488, 531, 515, 519, 492, 484, 480, 509, 542, 512, 488, 479, 488, 526, 465, 525, 508] payload_rows_per_dst=[10920, 10866, 10896, 10962, 10976, 10964, 10978, 10900] dup_rows_per_dst=[5628, 5478, 5380, 5442, 5456, 5546, 5454, 5226] combine_rows_per_src=[10920, 10866, 10896, 10962, 10976, 10964, 10978, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=458416 combine_payload_bytes=626917376 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627375792 ingress_bytes=777912320 link_bytes=777912320 token_bytes=1254293168 weight_movement_bytes=150994944 total_nvlink_bytes=1405288112 weight_to_token_ratio=0.120382 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.865023212e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87382 payload_rows_remote=76440 dup_rows=43690 dup_rows_remote=38144 dedup_ratio=0.333328 router_tokens_per_expert=[506, 523, 505, 506, 510, 509, 495, 483, 466, 506, 522, 569, 511, 522, 508, 489, 485, 540, 508, 503, 485, 497, 520, 510, 497, 496, 529, 474, 533, 490, 507, 542, 500, 514, 545, 551, 500, 507, 494, 508, 529, 529, 524, 507, 523, 474, 480, 475, 487, 501, 543, 534, 514, 537, 495, 590, 500, 513, 491, 520, 516, 521, 508, 523, 517, 504, 482, 509, 489, 508, 503, 524, 526, 489, 504, 534, 515, 516, 533, 525, 510, 537, 502, 490, 521, 535, 516, 484, 520, 536, 513, 512, 522, 499, 513, 527, 522, 481, 525, 510, 533, 486, 483, 501, 546, 518, 546, 484, 521, 550, 520, 518, 512, 505, 511, 515, 508, 532, 497, 522, 511, 462, 520, 507, 543, 490, 525, 518] payload_rows_per_dst=[10912, 10878, 11004, 10898, 10866, 10972, 10942, 10910] dup_rows_per_dst=[5348, 5354, 5316, 5688, 5490, 5502, 5546, 5446] combine_rows_per_src=[10912, 10878, 11004, 10898, 10866, 10972, 10942, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626475008 dispatch_meta_bytes=458496 combine_payload_bytes=626475008 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626933504 ingress_bytes=777469952 link_bytes=777469952 token_bytes=1253408512 weight_movement_bytes=150994944 total_nvlink_bytes=1404403456 weight_to_token_ratio=0.120467 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.859413363e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626688000 dispatch_meta_bytes=458672 combine_payload_bytes=626688000 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627146672 ingress_bytes=777682944 link_bytes=777682944 token_bytes=1253834672 weight_movement_bytes=150994944 total_nvlink_bytes=1404829616 weight_to_token_ratio=0.120427 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.862114401e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76606 dup_rows=43598 dup_rows_remote=37932 dedup_ratio=0.332626 router_tokens_per_expert=[562, 497, 509, 512, 529, 488, 429, 521, 538, 533, 549, 501, 507, 534, 475, 480, 519, 490, 498, 519, 503, 520, 550, 520, 497, 542, 539, 518, 552, 532, 519, 494, 503, 556, 549, 510, 517, 490, 538, 493, 536, 519, 509, 506, 504, 490, 508, 478, 496, 499, 482, 460, 515, 536, 526, 502, 533, 530, 513, 545, 483, 531, 477, 511, 506, 487, 517, 530, 493, 529, 512, 505, 486, 537, 524, 516, 506, 488, 509, 519, 511, 503, 550, 488, 528, 499, 536, 529, 464, 501, 514, 486, 518, 511, 502, 506, 510, 499, 494, 503, 511, 536, 528, 509, 507, 517, 522, 522, 495, 532, 496, 500, 509, 483, 485, 501, 519, 497, 551, 521, 515, 519, 541, 515, 540, 520, 540, 468] payload_rows_per_dst=[10996, 11020, 10884, 10870, 11056, 10890, 10890, 10868] dup_rows_per_dst=[5332, 5604, 5528, 5016, 5664, 5402, 5386, 5666] combine_rows_per_src=[10996, 11020, 10884, 10870, 11056, 10890, 10890, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87610 payload_rows_remote=76684 dup_rows=43462 dup_rows_remote=38110 dedup_ratio=0.331589 router_tokens_per_expert=[490, 498, 523, 522, 531, 504, 516, 489, 497, 526, 523, 511, 468, 499, 533, 508, 514, 530, 497, 492, 516, 514, 486, 482, 526, 510, 549, 500, 482, 529, 530, 517, 509, 484, 482, 522, 491, 544, 487, 527, 477, 512, 527, 502, 504, 503, 541, 527, 547, 527, 565, 505, 495, 470, 531, 505, 502, 520, 502, 465, 531, 507, 526, 512, 496, 497, 509, 538, 519, 555, 497, 503, 537, 469, 533, 504, 484, 471, 529, 511, 507, 484, 500, 522, 540, 499, 505, 543, 513, 553, 529, 501, 509, 567, 503, 516, 513, 546, 504, 496, 527, 519, 504, 535, 503, 528, 540, 493, 505, 514, 480, 531, 480, 531, 497, 512, 496, 511, 530, 527, 516, 508, 480, 500, 544, 541, 509, 512] payload_rows_per_dst=[10916, 10916, 10926, 10954, 10950, 11050, 10992, 10906] dup_rows_per_dst=[5360, 5432, 5352, 5466, 5354, 5532, 5484, 5482] combine_rows_per_src=[10916, 10916, 10926, 10954, 10950, 11050, 10992, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87378 payload_rows_remote=76406 dup_rows=43694 dup_rows_remote=38226 dedup_ratio=0.333359 router_tokens_per_expert=[495, 481, 496, 459, 530, 500, 504, 514, 528, 509, 499, 494, 523, 511, 515, 555, 484, 542, 474, 517, 590, 510, 497, 519, 494, 473, 538, 495, 503, 556, 508, 500, 533, 506, 550, 540, 515, 498, 490, 475, 542, 494, 516, 507, 504, 511, 508, 536, 544, 527, 509, 501, 501, 498, 541, 531, 510, 498, 550, 517, 493, 508, 539, 487, 469, 532, 525, 498, 562, 489, 502, 489, 537, 471, 503, 519, 536, 509, 550, 529, 521, 511, 564, 503, 555, 494, 500, 481, 510, 516, 489, 518, 547, 490, 443, 510, 537, 519, 485, 524, 522, 522, 534, 493, 521, 487, 504, 528, 511, 524, 514, 505, 516, 486, 514, 561, 498, 538, 504, 472, 536, 470, 523, 509, 510, 490, 503, 512] payload_rows_per_dst=[10854, 10898, 10970, 10930, 10972, 10886, 10974, 10894] dup_rows_per_dst=[5372, 5502, 5480, 5578, 5468, 5418, 5486, 5390] combine_rows_per_src=[10854, 10898, 10970, 10930, 10972, 10886, 10974, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626196480 dispatch_meta_bytes=458336 combine_payload_bytes=626196480 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626654816 ingress_bytes=852688896 link_bytes=852688896 token_bytes=1252851296 weight_movement_bytes=226492416 total_nvlink_bytes=1479343712 weight_to_token_ratio=0.180782 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081329545e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87226 payload_rows_remote=76138 dup_rows=43846 dup_rows_remote=38238 dedup_ratio=0.334518 router_tokens_per_expert=[516, 505, 500, 491, 510, 448, 476, 480, 453, 516, 568, 517, 518, 481, 489, 489, 488, 532, 547, 511, 526, 509, 533, 491, 555, 502, 492, 527, 525, 523, 530, 504, 486, 481, 508, 535, 566, 476, 495, 534, 539, 508, 501, 500, 581, 502, 521, 482, 511, 513, 496, 515, 516, 515, 500, 493, 525, 518, 507, 538, 496, 514, 515, 541, 499, 489, 491, 486, 505, 480, 513, 499, 512, 518, 493, 521, 467, 525, 516, 504, 518, 533, 518, 526, 510, 510, 525, 483, 508, 492, 523, 544, 484, 545, 565, 564, 511, 531, 508, 501, 527, 525, 489, 552, 516, 492, 526, 532, 503, 496, 472, 514, 505, 495, 563, 510, 516, 495, 528, 518, 516, 540, 487, 528, 523, 501, 512, 558] payload_rows_per_dst=[10710, 10972, 10880, 10890, 10746, 11088, 10904, 11036] dup_rows_per_dst=[5204, 5618, 5550, 5536, 5290, 5608, 5486, 5554] combine_rows_per_src=[10710, 10972, 10880, 10890, 10746, 11088, 10904, 11036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627556352 dispatch_meta_bytes=458152 combine_payload_bytes=627556352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628014504 ingress_bytes=703053824 link_bytes=703053824 token_bytes=1255570856 weight_movement_bytes=75497472 total_nvlink_bytes=1331068328 weight_to_token_ratio=0.060130 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915712111e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87282 payload_rows_remote=76012 dup_rows=43790 dup_rows_remote=37694 dedup_ratio=0.334091 router_tokens_per_expert=[543, 501, 504, 483, 511, 524, 499, 483, 498, 472, 501, 502, 569, 531, 521, 515, 505, 526, 540, 478, 510, 515, 506, 501, 547, 473, 488, 495, 502, 496, 561, 538, 503, 524, 458, 552, 514, 535, 513, 536, 483, 525, 483, 538, 470, 519, 527, 492, 477, 487, 512, 512, 506, 495, 520, 515, 502, 495, 534, 482, 531, 499, 512, 522, 500, 471, 554, 554, 514, 514, 517, 518, 507, 510, 508, 531, 504, 541, 527, 502, 510, 498, 488, 515, 537, 540, 526, 527, 511, 545, 497, 513, 503, 538, 525, 508, 519, 492, 494, 509, 500, 525, 579, 508, 526, 550, 567, 512, 536, 511, 502, 483, 539, 508, 520, 507, 507, 512, 502, 447, 506, 472, 528, 501, 536, 500, 514, 460] payload_rows_per_dst=[11270, 10630, 11132, 10904, 10882, 10752, 10912, 10800] dup_rows_per_dst=[6096, 5106, 5608, 5496, 5662, 5216, 5288, 5318] combine_rows_per_src=[11270, 10630, 11132, 10904, 10882, 10752, 10912, 10800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628195328 dispatch_meta_bytes=459176 combine_payload_bytes=628195328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628654504 ingress_bytes=703692800 link_bytes=703692800 token_bytes=1256849832 weight_movement_bytes=75497472 total_nvlink_bytes=1332347304 weight_to_token_ratio=0.060069 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923815227e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625917952 dispatch_meta_bytes=458528 combine_payload_bytes=625917952 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626376480 ingress_bytes=701415424 link_bytes=701415424 token_bytes=1252294432 weight_movement_bytes=75497472 total_nvlink_bytes=1327791904 weight_to_token_ratio=0.060287 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894934893e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87376 payload_rows_remote=76438 dup_rows=43696 dup_rows_remote=38290 dedup_ratio=0.333374 router_tokens_per_expert=[536, 513, 525, 522, 513, 455, 517, 516, 468, 505, 509, 523, 513, 509, 494, 503, 525, 472, 559, 517, 518, 490, 520, 495, 497, 494, 506, 500, 522, 515, 542, 500, 511, 527, 535, 511, 507, 485, 485, 557, 518, 515, 489, 485, 525, 482, 513, 523, 536, 492, 519, 502, 542, 530, 495, 489, 524, 533, 508, 510, 497, 544, 512, 502, 490, 550, 504, 505, 531, 554, 488, 533, 502, 480, 523, 546, 527, 533, 542, 484, 517, 539, 532, 526, 490, 535, 470, 534, 525, 485, 503, 488, 498, 536, 510, 555, 531, 506, 511, 497, 515, 505, 523, 532, 507, 503, 537, 514, 534, 551, 463, 490, 523, 497, 508, 463, 535, 546, 477, 517, 539, 493, 483, 529, 447, 521, 508, 500] payload_rows_per_dst=[10864, 10938, 10952, 10900, 11058, 10886, 10944, 10834] dup_rows_per_dst=[5378, 5406, 5384, 5570, 5526, 5600, 5494, 5338] combine_rows_per_src=[10864, 10938, 10952, 10900, 11058, 10886, 10944, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=623722496 dispatch_meta_bytes=457504 combine_payload_bytes=623722496 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=624180000 ingress_bytes=850214912 link_bytes=850214912 token_bytes=1247902496 weight_movement_bytes=226492416 total_nvlink_bytes=1474394912 weight_to_token_ratio=0.181498 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078192185e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76636 dup_rows=43554 dup_rows_remote=37994 dedup_ratio=0.332291 router_tokens_per_expert=[499, 505, 502, 507, 487, 503, 487, 538, 526, 514, 550, 508, 534, 517, 478, 498, 536, 545, 531, 518, 505, 487, 556, 524, 504, 532, 516, 568, 514, 507, 469, 535, 519, 506, 495, 517, 446, 502, 513, 487, 545, 514, 526, 518, 501, 529, 514, 566, 513, 512, 546, 522, 526, 489, 530, 512, 525, 509, 491, 495, 463, 526, 526, 536, 496, 533, 483, 514, 540, 474, 482, 539, 481, 520, 532, 516, 498, 505, 495, 517, 525, 504, 522, 531, 520, 519, 482, 509, 536, 511, 531, 467, 504, 459, 511, 558, 532, 452, 516, 549, 502, 491, 504, 506, 525, 507, 551, 454, 490, 512, 511, 514, 534, 495, 504, 506, 522, 521, 507, 503, 476, 512, 504, 502, 568, 493, 512, 528] payload_rows_per_dst=[10892, 11120, 10848, 10882, 10932, 10946, 10916, 10982] dup_rows_per_dst=[5414, 5574, 5548, 5560, 5318, 5432, 5316, 5392] combine_rows_per_src=[10892, 11120, 10848, 10882, 10932, 10946, 10916, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=622690304 dispatch_meta_bytes=454824 combine_payload_bytes=622690304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623145128 ingress_bytes=698187776 link_bytes=698187776 token_bytes=1245835432 weight_movement_bytes=75497472 total_nvlink_bytes=1321332904 weight_to_token_ratio=0.060600 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.854003773e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87126 payload_rows_remote=76260 dup_rows=43946 dup_rows_remote=38474 dedup_ratio=0.335281 router_tokens_per_expert=[517, 452, 523, 537, 498, 478, 501, 484, 483, 513, 506, 508, 493, 499, 532, 517, 526, 504, 535, 486, 503, 506, 527, 576, 502, 546, 511, 493, 484, 529, 491, 511, 499, 535, 516, 513, 545, 514, 475, 509, 552, 484, 558, 462, 503, 499, 519, 540, 506, 511, 510, 497, 517, 520, 506, 513, 503, 517, 505, 500, 529, 502, 493, 496, 510, 505, 555, 515, 500, 520, 537, 508, 526, 516, 534, 487, 504, 501, 494, 542, 525, 518, 492, 497, 540, 544, 533, 479, 502, 474, 535, 545, 503, 525, 506, 526, 514, 481, 511, 516, 500, 492, 502, 540, 530, 512, 503, 536, 501, 497, 530, 504, 522, 528, 500, 465, 493, 530, 547, 507, 514, 515, 523, 508, 531, 500, 536, 531] payload_rows_per_dst=[10828, 10986, 10866, 10812, 10892, 10934, 10866, 10942] dup_rows_per_dst=[5254, 5474, 5580, 5438, 5616, 5554, 5472, 5558] combine_rows_per_src=[10828, 10986, 10866, 10812, 10892, 10934, 10866, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626180096 dispatch_meta_bytes=458912 combine_payload_bytes=626180096 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626639008 ingress_bytes=852672512 link_bytes=852672512 token_bytes=1252819104 weight_movement_bytes=226492416 total_nvlink_bytes=1479311520 weight_to_token_ratio=0.180786 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081308768e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87088 payload_rows_remote=75986 dup_rows=43984 dup_rows_remote=38118 dedup_ratio=0.335571 router_tokens_per_expert=[536, 556, 457, 499, 573, 504, 522, 500, 523, 502, 509, 509, 490, 518, 565, 526, 510, 508, 484, 474, 506, 534, 494, 536, 487, 536, 551, 484, 555, 524, 530, 528, 485, 519, 519, 510, 530, 496, 552, 520, 510, 527, 543, 504, 482, 520, 533, 507, 507, 529, 512, 516, 547, 493, 508, 521, 527, 500, 489, 508, 513, 505, 513, 495, 505, 514, 512, 538, 536, 529, 509, 487, 525, 532, 534, 491, 521, 526, 520, 522, 508, 490, 494, 481, 508, 497, 467, 516, 511, 521, 513, 473, 466, 504, 538, 516, 531, 505, 525, 526, 520, 515, 541, 501, 478, 487, 514, 506, 457, 497, 507, 528, 509, 481, 492, 515, 576, 497, 516, 513, 511, 533, 497, 510, 499, 494, 509, 472] payload_rows_per_dst=[10590, 10722, 11046, 10874, 11090, 10866, 10798, 11102] dup_rows_per_dst=[5268, 5244, 5468, 5492, 5734, 5434, 5478, 5866] combine_rows_per_src=[10590, 10722, 11046, 10874, 11090, 10866, 10798, 11102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=458520 combine_payload_bytes=627802112 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628260632 ingress_bytes=778797056 link_bytes=778797056 token_bytes=1256062744 weight_movement_bytes=150994944 total_nvlink_bytes=1407057688 weight_to_token_ratio=0.120213 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876242910e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87624 payload_rows_remote=76740 dup_rows=43448 dup_rows_remote=37946 dedup_ratio=0.331482 router_tokens_per_expert=[519, 517, 492, 521, 562, 515, 507, 518, 476, 522, 525, 534, 508, 509, 537, 487, 562, 543, 512, 530, 499, 519, 513, 496, 502, 532, 493, 509, 510, 519, 513, 522, 509, 536, 557, 474, 504, 516, 537, 503, 467, 502, 472, 515, 532, 557, 520, 492, 509, 473, 495, 457, 516, 488, 537, 554, 529, 516, 536, 518, 517, 518, 509, 521, 497, 519, 467, 519, 552, 504, 490, 491, 496, 501, 500, 534, 500, 527, 468, 484, 510, 521, 526, 530, 529, 537, 507, 521, 499, 524, 560, 573, 503, 483, 489, 533, 501, 528, 520, 514, 516, 491, 503, 489, 506, 500, 524, 552, 498, 502, 533, 498, 515, 509, 513, 478, 536, 525, 521, 456, 508, 550, 502, 481, 496, 491, 507, 470] payload_rows_per_dst=[11026, 11022, 10884, 10970, 10860, 11064, 10878, 10920] dup_rows_per_dst=[5472, 5526, 5502, 5416, 5238, 5626, 5472, 5196] combine_rows_per_src=[11026, 11022, 10884, 10970, 10860, 11064, 10878, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87604 payload_rows_remote=76652 dup_rows=43468 dup_rows_remote=38132 dedup_ratio=0.331635 router_tokens_per_expert=[494, 499, 499, 533, 559, 514, 537, 471, 555, 513, 498, 526, 538, 514, 478, 527, 495, 498, 506, 507, 513, 504, 502, 509, 528, 517, 477, 518, 519, 540, 480, 541, 490, 515, 490, 512, 512, 495, 535, 498, 545, 447, 502, 489, 527, 533, 501, 529, 472, 476, 573, 542, 546, 520, 540, 535, 475, 562, 472, 528, 509, 509, 499, 533, 550, 539, 519, 523, 504, 502, 525, 527, 502, 489, 469, 447, 509, 501, 508, 530, 519, 566, 538, 514, 483, 474, 539, 511, 538, 537, 497, 531, 536, 554, 505, 508, 464, 485, 493, 477, 527, 485, 526, 480, 517, 529, 508, 520, 518, 538, 507, 498, 508, 516, 501, 502, 484, 520, 540, 547, 538, 464, 517, 501, 449, 524, 527, 512] payload_rows_per_dst=[11000, 10948, 10788, 11068, 10952, 11082, 10844, 10922] dup_rows_per_dst=[5510, 5360, 5452, 5514, 5336, 5618, 5300, 5378] combine_rows_per_src=[11000, 10948, 10788, 11068, 10952, 11082, 10844, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=624721920 dispatch_meta_bytes=458936 combine_payload_bytes=624721920 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=625180856 ingress_bytes=851214336 link_bytes=851214336 token_bytes=1249902776 weight_movement_bytes=226492416 total_nvlink_bytes=1476395192 weight_to_token_ratio=0.181208 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079459596e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15068953168 prefetch_bytes=905969664 reduce_bytes=528482304 weight_movement_bytes=1434451968 egress_bytes=7537228368 ingress_bytes=8966176768 link_bytes=8966176768 total_nvlink_bytes=16503405136 weight_to_token_ratio=0.095193 prefetch_share_of_ingress=0.101043 dedup_ratio=0.331846 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=7 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.475311856e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=622477312 dispatch_meta_bytes=456416 combine_payload_bytes=622477312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=622933728 ingress_bytes=697974784 link_bytes=697974784 token_bytes=1245411040 weight_movement_bytes=75497472 total_nvlink_bytes=1320908512 weight_to_token_ratio=0.060621 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.851302735e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76722 dup_rows=43520 dup_rows_remote=38122 dedup_ratio=0.332031 router_tokens_per_expert=[515, 506, 530, 524, 511, 526, 507, 527, 524, 525, 516, 491, 527, 530, 502, 556, 520, 555, 491, 515, 519, 538, 473, 487, 539, 494, 510, 514, 505, 525, 557, 492, 451, 540, 466, 549, 510, 485, 528, 522, 492, 511, 527, 509, 524, 545, 506, 555, 491, 511, 504, 501, 499, 499, 485, 496, 507, 502, 499, 544, 544, 530, 499, 500, 486, 483, 495, 521, 533, 492, 487, 516, 516, 505, 497, 501, 536, 485, 477, 547, 508, 507, 528, 528, 510, 524, 500, 507, 507, 531, 508, 473, 482, 513, 472, 516, 516, 519, 528, 502, 530, 523, 524, 537, 510, 513, 524, 544, 505, 544, 507, 482, 510, 499, 509, 531, 517, 507, 486, 510, 510, 498, 459, 520, 553, 518, 516, 512] payload_rows_per_dst=[11024, 11026, 11032, 10862, 10888, 10830, 10968, 10922] dup_rows_per_dst=[5610, 5442, 5408, 5360, 5266, 5398, 5648, 5388] combine_rows_per_src=[11024, 11026, 11032, 10862, 10888, 10830, 10968, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87630 payload_rows_remote=76640 dup_rows=43442 dup_rows_remote=37890 dedup_ratio=0.331436 router_tokens_per_expert=[508, 509, 531, 480, 524, 534, 496, 519, 511, 531, 499, 477, 509, 522, 494, 524, 510, 514, 531, 533, 505, 530, 569, 551, 493, 516, 530, 522, 502, 480, 476, 538, 517, 498, 503, 475, 492, 558, 494, 574, 473, 491, 533, 510, 550, 508, 527, 480, 502, 512, 515, 534, 506, 497, 494, 558, 518, 515, 522, 524, 505, 505, 531, 514, 552, 527, 504, 492, 500, 486, 522, 490, 504, 499, 510, 519, 493, 507, 534, 519, 495, 507, 520, 511, 548, 493, 503, 496, 492, 529, 477, 515, 493, 528, 492, 504, 541, 495, 522, 521, 484, 507, 517, 520, 533, 506, 537, 523, 521, 502, 542, 502, 543, 527, 517, 504, 486, 520, 481, 472, 509, 492, 494, 454, 520, 525, 526, 529] payload_rows_per_dst=[10990, 11160, 11050, 10910, 10894, 10908, 10868, 10850] dup_rows_per_dst=[5552, 5474, 5564, 5430, 5422, 5298, 5354, 5348] combine_rows_per_src=[10990, 11160, 11050, 10910, 10894, 10908, 10868, 10850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628654080 dispatch_meta_bytes=458744 combine_payload_bytes=628654080 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629112824 ingress_bytes=704151552 link_bytes=704151552 token_bytes=1257766904 weight_movement_bytes=75497472 total_nvlink_bytes=1333264376 weight_to_token_ratio=0.060025 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.929632848e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627933184 dispatch_meta_bytes=459136 combine_payload_bytes=627933184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628392320 ingress_bytes=703430656 link_bytes=703430656 token_bytes=1256325504 weight_movement_bytes=75497472 total_nvlink_bytes=1331822976 weight_to_token_ratio=0.060094 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920490872e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15042380640 prefetch_bytes=679477248 reduce_bytes=1434451968 weight_movement_bytes=2113929216 egress_bytes=7523943264 ingress_bytes=9632366592 link_bytes=9632366592 total_nvlink_bytes=17156309856 weight_to_token_ratio=0.140532 prefetch_share_of_ingress=0.070541 dedup_ratio=0.333066 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=19 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.017933058e-04 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15034402944 prefetch_bytes=905969664 reduce_bytes=150994944 weight_movement_bytes=1056964608 egress_bytes=7519946880 ingress_bytes=8571420672 link_bytes=8571420672 total_nvlink_bytes=16091367552 weight_to_token_ratio=0.070303 prefetch_share_of_ingress=0.105697 dedup_ratio=0.332447 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=2 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.058139942e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628506624 dispatch_meta_bytes=459376 combine_payload_bytes=628506624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628966000 ingress_bytes=704004096 link_bytes=704004096 token_bytes=1257472624 weight_movement_bytes=75497472 total_nvlink_bytes=1332970096 weight_to_token_ratio=0.060039 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927762898e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76540 dup_rows=43654 dup_rows_remote=38076 dedup_ratio=0.333054 router_tokens_per_expert=[528, 506, 578, 548, 509, 501, 528, 527, 543, 476, 574, 484, 519, 471, 509, 552, 554, 516, 509, 496, 477, 527, 486, 475, 520, 523, 535, 528, 507, 484, 542, 493, 523, 524, 533, 472, 516, 505, 498, 493, 480, 507, 495, 514, 505, 496, 494, 493, 496, 513, 504, 524, 537, 528, 506, 532, 500, 492, 462, 551, 529, 538, 539, 477, 528, 490, 456, 487, 558, 517, 492, 535, 530, 530, 480, 523, 507, 548, 482, 487, 501, 508, 477, 510, 499, 491, 515, 518, 495, 501, 496, 470, 533, 453, 512, 508, 519, 518, 535, 528, 515, 493, 510, 535, 527, 519, 510, 509, 532, 524, 531, 539, 519, 514, 547, 510, 524, 483, 526, 542, 527, 480, 544, 501, 517, 520, 501, 499] payload_rows_per_dst=[11000, 10830, 10778, 10878, 10988, 10868, 11054, 11022] dup_rows_per_dst=[5706, 5514, 5318, 5578, 5312, 5106, 5634, 5486] combine_rows_per_src=[11000, 10830, 10778, 10878, 10988, 10868, 11054, 11022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15058728216 prefetch_bytes=603979776 reduce_bytes=1283457024 weight_movement_bytes=1887436800 egress_bytes=7532115224 ingress_bytes=9414049792 link_bytes=9414049792 total_nvlink_bytes=16946165016 weight_to_token_ratio=0.125338 prefetch_share_of_ingress=0.064157 dedup_ratio=0.332127 prefetch_slots=16 unused_prefetch_slots=368 slot_utilization=0.041667 reduce_slots=17 remote_demand=8 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=26924 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.948616887e-05 prefetch_bytes_per_gemm_flop=6.382761438e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15051749496 prefetch_bytes=830472192 reduce_bytes=377487360 weight_movement_bytes=1207959552 egress_bytes=7528626296 ingress_bytes=8731082752 link_bytes=8731082752 total_nvlink_bytes=16259709048 weight_to_token_ratio=0.080254 prefetch_share_of_ingress=0.095117 dedup_ratio=0.332868 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=5 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.226868269e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627834880 dispatch_meta_bytes=458120 combine_payload_bytes=627834880 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628293000 ingress_bytes=778829824 link_bytes=778829824 token_bytes=1256127880 weight_movement_bytes=150994944 total_nvlink_bytes=1407122824 weight_to_token_ratio=0.120207 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876658454e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15055490648 prefetch_bytes=754974720 reduce_bytes=754974720 weight_movement_bytes=1509949440 egress_bytes=7530499672 ingress_bytes=9034940416 link_bytes=9034940416 total_nvlink_bytes=16565440088 weight_to_token_ratio=0.100292 prefetch_share_of_ingress=0.083562 dedup_ratio=0.332930 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=10 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.547980176e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=5 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=458464 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627474144 ingress_bytes=778010624 link_bytes=778010624 token_bytes=1254489824 weight_movement_bytes=150994944 total_nvlink_bytes=1405484768 weight_to_token_ratio=0.120364 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866269845e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15011659128 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=7508573560 ingress_bytes=9164029952 link_bytes=9164029952 total_nvlink_bytes=16672603512 weight_to_token_ratio=0.110644 prefetch_share_of_ingress=0.090623 dedup_ratio=0.333304 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.684399928e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=5 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15057615544 prefetch_bytes=830472192 reduce_bytes=981467136 weight_movement_bytes=1811939328 egress_bytes=7531559608 ingress_bytes=9337995264 link_bytes=9337995264 total_nvlink_bytes=16869554872 weight_to_token_ratio=0.120334 prefetch_share_of_ingress=0.088935 dedup_ratio=0.332400 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=13 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043658 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.868243681e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none + [2026-08-29 23:36:23] iteration 6/ 12 | consumed samples: 48 | elapsed time per iteration (ms): 358.5 | throughput per GPU (TFLOP/s/GPU): 504.4 | learning rate: 7.750000E-05 | global batch size: 8 | lm loss: 1.014113E+01 | loss scale: 1.0 | grad norm: 27.606 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87220 payload_rows_remote=76386 dup_rows=43852 dup_rows_remote=38372 dedup_ratio=0.334564 router_tokens_per_expert=[525, 508, 505, 483, 526, 468, 566, 526, 525, 521, 551, 536, 525, 522, 508, 550, 473, 522, 506, 504, 491, 504, 505, 506, 507, 488, 523, 531, 485, 495, 533, 490, 513, 534, 521, 484, 505, 506, 489, 498, 468, 500, 474, 556, 547, 523, 533, 506, 452, 518, 508, 517, 521, 539, 538, 525, 482, 479, 504, 514, 530, 494, 522, 503, 538, 491, 520, 497, 530, 532, 508, 523, 528, 543, 525, 519, 549, 527, 531, 535, 481, 506, 533, 522, 505, 488, 505, 500, 491, 523, 542, 513, 500, 548, 461, 539, 509, 447, 543, 533, 499, 497, 529, 527, 478, 523, 514, 502, 498, 520, 518, 512, 511, 487, 480, 499, 492, 528, 518, 491, 518, 533, 504, 511, 511, 499, 533, 508] payload_rows_per_dst=[10974, 10908, 10834, 10888, 10996, 10850, 10948, 10822] dup_rows_per_dst=[5716, 5218, 5480, 5404, 5796, 5464, 5350, 5424] combine_rows_per_src=[10974, 10908, 10834, 10888, 10996, 10850, 10948, 10822] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87612 payload_rows_remote=76664 dup_rows=43460 dup_rows_remote=37886 dedup_ratio=0.331573 router_tokens_per_expert=[550, 543, 511, 517, 518, 505, 541, 465, 496, 546, 558, 532, 525, 516, 516, 515, 519, 523, 567, 492, 488, 534, 507, 512, 544, 456, 525, 541, 494, 509, 519, 531, 508, 506, 517, 524, 513, 483, 519, 484, 498, 539, 509, 496, 553, 490, 496, 515, 503, 507, 532, 548, 514, 467, 504, 494, 512, 514, 514, 517, 525, 532, 495, 562, 554, 496, 491, 514, 505, 505, 537, 499, 493, 517, 547, 518, 467, 508, 496, 512, 511, 509, 491, 495, 503, 519, 524, 524, 494, 534, 505, 486, 525, 492, 494, 546, 508, 474, 509, 532, 493, 490, 534, 489, 529, 467, 488, 498, 489, 515, 522, 533, 550, 533, 463, 495, 481, 542, 495, 511, 510, 544, 522, 470, 488, 521, 512, 513] payload_rows_per_dst=[11000, 10948, 10996, 10966, 10868, 11000, 10868, 10966] dup_rows_per_dst=[5708, 5574, 5304, 5514, 5450, 5304, 5272, 5334] combine_rows_per_src=[11000, 10948, 10996, 10966, 10868, 11000, 10868, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87304 payload_rows_remote=76318 dup_rows=43768 dup_rows_remote=38412 dedup_ratio=0.333923 router_tokens_per_expert=[526, 445, 508, 504, 487, 511, 560, 518, 532, 520, 504, 547, 557, 527, 510, 519, 518, 486, 524, 475, 493, 530, 509, 511, 552, 493, 539, 542, 496, 506, 555, 542, 513, 537, 526, 495, 504, 519, 480, 516, 507, 515, 522, 524, 482, 511, 501, 548, 508, 525, 518, 493, 527, 530, 484, 494, 517, 496, 510, 528, 529, 498, 455, 516, 520, 519, 505, 507, 492, 549, 496, 572, 511, 507, 508, 499, 503, 541, 488, 480, 552, 524, 498, 539, 507, 480, 520, 495, 519, 512, 492, 488, 524, 486, 521, 513, 494, 507, 547, 508, 518, 510, 540, 523, 515, 506, 489, 524, 537, 497, 514, 482, 540, 520, 531, 533, 452, 524, 539, 502, 469, 493, 532, 484, 507, 500, 483, 475] payload_rows_per_dst=[10436, 10746, 11290, 10812, 10950, 10976, 11108, 10986] dup_rows_per_dst=[4782, 5362, 5974, 5498, 5566, 5526, 5704, 5356] combine_rows_per_src=[10436, 10746, 11290, 10812, 10950, 10976, 11108, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76450 dup_rows=43648 dup_rows_remote=38002 dedup_ratio=0.333008 router_tokens_per_expert=[525, 533, 520, 519, 510, 543, 524, 527, 524, 548, 526, 498, 508, 512, 497, 510, 545, 522, 471, 519, 489, 515, 538, 505, 515, 504, 485, 543, 558, 499, 501, 505, 526, 491, 490, 507, 505, 542, 486, 477, 505, 523, 470, 489, 488, 510, 492, 504, 510, 525, 538, 529, 510, 488, 482, 552, 507, 492, 527, 502, 519, 534, 544, 551, 507, 495, 515, 518, 499, 518, 519, 491, 500, 501, 545, 504, 547, 472, 523, 521, 511, 496, 540, 514, 491, 494, 521, 500, 535, 508, 518, 522, 477, 505, 506, 477, 520, 503, 526, 534, 471, 535, 527, 491, 520, 531, 497, 489, 511, 508, 498, 505, 518, 495, 489, 490, 497, 487, 571, 558, 463, 517, 492, 516, 509, 517, 541, 567] payload_rows_per_dst=[11070, 11004, 10760, 10974, 10902, 10886, 10926, 10902] dup_rows_per_dst=[5578, 5424, 5250, 5646, 5448, 5344, 5406, 5552] combine_rows_per_src=[11070, 11004, 10760, 10974, 10902, 10886, 10926, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76564 dup_rows=43666 dup_rows_remote=38210 dedup_ratio=0.333145 router_tokens_per_expert=[492, 525, 527, 487, 481, 535, 527, 493, 526, 494, 465, 499, 520, 514, 527, 494, 480, 522, 486, 495, 542, 510, 486, 521, 470, 532, 518, 505, 513, 544, 548, 543, 487, 526, 502, 537, 507, 497, 515, 498, 515, 504, 489, 485, 510, 511, 528, 540, 497, 504, 500, 531, 517, 500, 495, 494, 523, 507, 509, 546, 479, 504, 528, 487, 515, 512, 511, 499, 512, 501, 454, 492, 544, 496, 537, 493, 515, 515, 539, 536, 540, 446, 519, 536, 528, 509, 485, 533, 540, 493, 515, 500, 478, 530, 539, 511, 527, 531, 501, 527, 497, 503, 548, 518, 546, 500, 485, 543, 461, 492, 466, 504, 507, 528, 529, 528, 545, 505, 518, 500, 514, 515, 573, 494, 505, 574, 540, 546] payload_rows_per_dst=[10934, 10894, 10898, 10886, 10830, 11028, 10842, 11094] dup_rows_per_dst=[5278, 5536, 5404, 5356, 5512, 5376, 5456, 5748] combine_rows_per_src=[10934, 10894, 10898, 10886, 10830, 11028, 10842, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=459032 combine_payload_bytes=625754112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626213144 ingress_bytes=701251584 link_bytes=701251584 token_bytes=1251967256 weight_movement_bytes=75497472 total_nvlink_bytes=1327464728 weight_to_token_ratio=0.060303 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892857171e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87512 payload_rows_remote=76568 dup_rows=43560 dup_rows_remote=38132 dedup_ratio=0.332336 router_tokens_per_expert=[553, 513, 559, 496, 489, 496, 512, 544, 494, 524, 518, 498, 517, 535, 504, 533, 518, 498, 540, 509, 501, 522, 514, 475, 501, 482, 534, 553, 512, 537, 493, 488, 548, 475, 532, 534, 482, 473, 516, 514, 509, 477, 517, 535, 513, 488, 499, 512, 521, 496, 482, 515, 480, 518, 527, 494, 543, 524, 507, 513, 510, 510, 487, 513, 504, 536, 504, 466, 495, 540, 553, 511, 498, 500, 538, 504, 490, 508, 533, 506, 511, 507, 505, 485, 499, 530, 504, 508, 542, 519, 524, 543, 540, 472, 520, 494, 514, 542, 540, 525, 515, 492, 538, 477, 561, 524, 534, 525, 481, 492, 505, 497, 521, 530, 487, 464, 497, 518, 514, 502, 541, 546, 498, 475, 502, 530, 497, 537] payload_rows_per_dst=[11022, 10910, 10934, 10866, 10944, 10834, 11120, 10882] dup_rows_per_dst=[5548, 5444, 5314, 5414, 5428, 5572, 5404, 5436] combine_rows_per_src=[11022, 10910, 10934, 10866, 10944, 10834, 11120, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628031488 dispatch_meta_bytes=458200 combine_payload_bytes=628031488 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=628489688 ingress_bytes=930021376 link_bytes=930021376 token_bytes=1256521176 weight_movement_bytes=301989888 total_nvlink_bytes=1558511064 weight_to_token_ratio=0.240338 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.179398015e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625197056 dispatch_meta_bytes=458920 combine_payload_bytes=625197056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625655976 ingress_bytes=700694528 link_bytes=700694528 token_bytes=1250853032 weight_movement_bytes=75497472 total_nvlink_bytes=1326350504 weight_to_token_ratio=0.060357 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.885792917e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626278400 dispatch_meta_bytes=457808 combine_payload_bytes=626278400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626736208 ingress_bytes=701775872 link_bytes=701775872 token_bytes=1253014608 weight_movement_bytes=75497472 total_nvlink_bytes=1328512080 weight_to_token_ratio=0.060253 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899505881e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627212288 dispatch_meta_bytes=459096 combine_payload_bytes=627212288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627671384 ingress_bytes=702709760 link_bytes=702709760 token_bytes=1254883672 weight_movement_bytes=75497472 total_nvlink_bytes=1330381144 weight_to_token_ratio=0.060163 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911348896e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87360 payload_rows_remote=76486 dup_rows=43712 dup_rows_remote=38294 dedup_ratio=0.333496 router_tokens_per_expert=[526, 490, 501, 510, 449, 527, 548, 526, 492, 505, 505, 512, 502, 522, 512, 519, 536, 479, 492, 485, 514, 493, 498, 505, 537, 555, 510, 506, 539, 543, 522, 556, 508, 539, 517, 507, 458, 538, 494, 520, 513, 539, 507, 505, 488, 505, 503, 520, 523, 484, 523, 516, 520, 530, 511, 500, 483, 514, 530, 548, 516, 500, 498, 501, 503, 496, 518, 491, 519, 517, 519, 471, 523, 509, 549, 524, 474, 490, 502, 463, 578, 525, 492, 535, 550, 511, 516, 544, 544, 542, 518, 494, 519, 514, 508, 509, 513, 519, 523, 536, 491, 509, 519, 493, 514, 530, 528, 499, 520, 535, 493, 533, 535, 520, 496, 513, 478, 542, 483, 523, 512, 486, 479, 479, 488, 493, 498, 515] payload_rows_per_dst=[10874, 11050, 10762, 10972, 10792, 11068, 10990, 10852] dup_rows_per_dst=[5418, 5490, 5560, 5422, 5344, 5730, 5520, 5228] combine_rows_per_src=[10874, 11050, 10762, 10972, 10792, 11068, 10990, 10852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627245056 dispatch_meta_bytes=458800 combine_payload_bytes=627245056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627703856 ingress_bytes=702742528 link_bytes=702742528 token_bytes=1254948912 weight_movement_bytes=75497472 total_nvlink_bytes=1330446384 weight_to_token_ratio=0.060160 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911764440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87292 payload_rows_remote=76442 dup_rows=43780 dup_rows_remote=38516 dedup_ratio=0.334015 router_tokens_per_expert=[529, 524, 520, 538, 526, 508, 528, 511, 555, 542, 492, 517, 513, 559, 505, 500, 533, 548, 486, 521, 535, 506, 541, 512, 526, 487, 558, 490, 525, 469, 535, 510, 487, 510, 473, 548, 498, 510, 504, 504, 500, 540, 518, 496, 500, 564, 499, 505, 512, 493, 504, 479, 475, 534, 529, 493, 527, 530, 556, 497, 521, 540, 541, 496, 470, 535, 542, 501, 498, 497, 500, 501, 485, 495, 541, 502, 515, 493, 533, 515, 517, 507, 476, 520, 557, 479, 502, 502, 472, 500, 522, 513, 502, 524, 510, 454, 546, 513, 500, 520, 523, 501, 487, 512, 503, 502, 495, 497, 482, 477, 482, 539, 533, 486, 508, 557, 537, 527, 533, 509, 557, 468, 499, 468, 540, 536, 511, 476] payload_rows_per_dst=[11100, 10888, 10854, 10906, 10782, 10850, 10884, 11028] dup_rows_per_dst=[5634, 5676, 5458, 5548, 5464, 5264, 5274, 5462] combine_rows_per_src=[11100, 10888, 10854, 10906, 10782, 10850, 10884, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87520 payload_rows_remote=76662 dup_rows=43552 dup_rows_remote=37934 dedup_ratio=0.332275 router_tokens_per_expert=[517, 530, 516, 483, 456, 488, 504, 506, 468, 531, 565, 545, 531, 503, 491, 533, 535, 491, 521, 507, 540, 492, 485, 521, 488, 509, 488, 545, 557, 490, 539, 492, 528, 552, 521, 528, 480, 508, 473, 529, 522, 558, 530, 515, 554, 477, 497, 466, 479, 516, 533, 521, 503, 522, 528, 539, 519, 515, 527, 491, 498, 512, 508, 516, 523, 521, 480, 543, 537, 549, 475, 514, 480, 497, 482, 510, 522, 498, 487, 538, 514, 519, 489, 517, 501, 555, 463, 510, 530, 514, 505, 524, 523, 524, 539, 488, 520, 481, 495, 511, 448, 528, 502, 518, 565, 564, 526, 523, 532, 511, 510, 503, 520, 489, 490, 511, 505, 500, 527, 504, 488, 483, 484, 522, 498, 527, 532, 516] payload_rows_per_dst=[10966, 11004, 10858, 10932, 10984, 10916, 10962, 10898] dup_rows_per_dst=[5368, 5396, 5618, 5522, 5328, 5514, 5512, 5294] combine_rows_per_src=[10966, 11004, 10858, 10932, 10984, 10916, 10962, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87282 payload_rows_remote=76380 dup_rows=43790 dup_rows_remote=38356 dedup_ratio=0.334091 router_tokens_per_expert=[528, 506, 553, 530, 521, 506, 516, 525, 528, 540, 520, 485, 447, 531, 529, 476, 526, 528, 539, 510, 500, 522, 486, 497, 522, 518, 504, 508, 488, 527, 538, 455, 516, 509, 538, 507, 551, 521, 515, 539, 500, 562, 519, 558, 502, 515, 483, 518, 518, 483, 533, 531, 509, 519, 537, 518, 508, 512, 474, 489, 493, 499, 529, 536, 499, 506, 531, 521, 507, 539, 490, 529, 481, 496, 488, 535, 511, 522, 502, 483, 518, 505, 509, 467, 528, 498, 498, 493, 501, 497, 545, 495, 502, 533, 475, 474, 507, 476, 494, 508, 541, 489, 563, 535, 541, 467, 482, 515, 511, 519, 530, 535, 534, 491, 467, 518, 516, 535, 533, 501, 525, 513, 529, 511, 470, 508, 511, 533] payload_rows_per_dst=[10912, 10902, 10994, 10948, 10912, 10742, 10944, 10928] dup_rows_per_dst=[5570, 5434, 5712, 5428, 5368, 5334, 5482, 5462] combine_rows_per_src=[10912, 10902, 10994, 10948, 10912, 10742, 10944, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87638 payload_rows_remote=76614 dup_rows=43434 dup_rows_remote=37844 dedup_ratio=0.331375 router_tokens_per_expert=[518, 508, 492, 516, 497, 517, 518, 509, 463, 528, 524, 482, 540, 507, 519, 542, 537, 506, 532, 526, 509, 519, 519, 513, 484, 512, 532, 500, 513, 476, 522, 484, 474, 481, 525, 526, 533, 504, 526, 506, 505, 520, 499, 520, 494, 490, 521, 483, 484, 508, 523, 514, 513, 537, 514, 510, 540, 515, 515, 526, 522, 520, 545, 516, 539, 517, 537, 501, 548, 519, 542, 508, 495, 484, 503, 482, 537, 486, 504, 529, 521, 518, 530, 499, 496, 502, 528, 508, 543, 505, 496, 531, 511, 509, 515, 491, 507, 528, 524, 536, 536, 518, 506, 478, 522, 525, 512, 492, 512, 492, 532, 519, 495, 528, 526, 462, 495, 551, 469, 485, 500, 514, 514, 509, 485, 515, 518, 524] payload_rows_per_dst=[10912, 10938, 10686, 10930, 11074, 11138, 10936, 11024] dup_rows_per_dst=[5448, 5430, 5064, 5264, 5396, 5678, 5564, 5590] combine_rows_per_src=[10912, 10938, 10686, 10930, 11074, 11138, 10936, 11024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87422 payload_rows_remote=76492 dup_rows=43650 dup_rows_remote=38276 dedup_ratio=0.333023 router_tokens_per_expert=[526, 503, 555, 536, 506, 520, 554, 496, 537, 493, 484, 495, 544, 496, 475, 514, 497, 514, 509, 508, 526, 526, 488, 503, 520, 510, 559, 519, 538, 543, 482, 521, 554, 505, 485, 497, 512, 488, 529, 527, 539, 508, 516, 506, 514, 532, 551, 502, 523, 495, 522, 503, 520, 527, 456, 526, 537, 497, 503, 487, 481, 554, 489, 532, 486, 529, 520, 526, 469, 526, 536, 550, 528, 483, 489, 505, 551, 480, 543, 502, 493, 535, 528, 509, 478, 565, 490, 507, 500, 521, 554, 495, 494, 513, 526, 479, 530, 487, 525, 503, 492, 508, 485, 462, 515, 471, 492, 507, 481, 528, 552, 513, 533, 492, 506, 482, 487, 481, 502, 487, 521, 528, 518, 515, 542, 520, 550, 497] payload_rows_per_dst=[11022, 10858, 10960, 10930, 10898, 10900, 10864, 10990] dup_rows_per_dst=[5446, 5668, 5570, 5374, 5548, 5474, 5238, 5332] combine_rows_per_src=[11022, 10858, 10960, 10930, 10898, 10900, 10864, 10990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87548 payload_rows_remote=76512 dup_rows=43524 dup_rows_remote=38052 dedup_ratio=0.332062 router_tokens_per_expert=[521, 484, 505, 495, 524, 543, 508, 537, 493, 562, 484, 568, 518, 476, 521, 499, 540, 541, 516, 493, 511, 532, 470, 499, 539, 541, 543, 499, 512, 532, 506, 499, 497, 517, 483, 519, 514, 533, 515, 533, 522, 508, 513, 529, 496, 514, 505, 510, 517, 494, 526, 512, 518, 523, 475, 491, 526, 516, 508, 528, 530, 471, 505, 548, 504, 522, 505, 494, 505, 515, 507, 455, 486, 529, 514, 531, 490, 496, 510, 545, 505, 507, 506, 524, 487, 513, 520, 522, 519, 524, 530, 507, 508, 526, 489, 459, 531, 500, 527, 534, 514, 498, 526, 532, 536, 502, 477, 534, 494, 469, 532, 548, 494, 530, 527, 513, 508, 493, 482, 506, 512, 499, 505, 500, 520, 482, 536, 514] payload_rows_per_dst=[11006, 11086, 10912, 10884, 10858, 10904, 11036, 10862] dup_rows_per_dst=[5470, 5460, 5504, 5492, 5358, 5388, 5472, 5380] combine_rows_per_src=[11006, 11086, 10912, 10884, 10858, 10904, 11036, 10862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626573312 dispatch_meta_bytes=459120 combine_payload_bytes=626573312 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=627032432 ingress_bytes=928563200 link_bytes=928563200 token_bytes=1253605744 weight_movement_bytes=301989888 total_nvlink_bytes=1555595632 weight_to_token_ratio=0.240897 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177548843e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626212864 dispatch_meta_bytes=459832 combine_payload_bytes=626212864 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626672696 ingress_bytes=701710336 link_bytes=701710336 token_bytes=1252885560 weight_movement_bytes=75497472 total_nvlink_bytes=1328383032 weight_to_token_ratio=0.060259 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898674792e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628015104 dispatch_meta_bytes=458384 combine_payload_bytes=628015104 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=628473488 ingress_bytes=930004992 link_bytes=930004992 token_bytes=1256488592 weight_movement_bytes=301989888 total_nvlink_bytes=1558478480 weight_to_token_ratio=0.240344 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.179377238e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625704960 dispatch_meta_bytes=458944 combine_payload_bytes=625704960 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626163904 ingress_bytes=701202432 link_bytes=701202432 token_bytes=1251868864 weight_movement_bytes=75497472 total_nvlink_bytes=1327366336 weight_to_token_ratio=0.060308 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892233855e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87572 payload_rows_remote=76474 dup_rows=43500 dup_rows_remote=37946 dedup_ratio=0.331879 router_tokens_per_expert=[496, 486, 514, 482, 512, 535, 500, 534, 540, 466, 487, 510, 510, 500, 513, 516, 491, 477, 551, 480, 489, 465, 542, 505, 543, 522, 489, 550, 482, 504, 522, 521, 496, 497, 539, 452, 550, 513, 471, 515, 484, 521, 550, 536, 524, 520, 494, 530, 512, 576, 480, 529, 467, 528, 527, 505, 531, 512, 495, 503, 494, 520, 491, 513, 501, 573, 521, 474, 505, 489, 539, 477, 544, 531, 554, 486, 540, 535, 530, 527, 469, 484, 492, 531, 544, 498, 532, 481, 485, 491, 521, 532, 513, 501, 490, 490, 521, 494, 534, 535, 534, 531, 522, 496, 513, 532, 512, 505, 500, 530, 511, 519, 491, 539, 565, 554, 510, 491, 491, 512, 533, 487, 517, 493, 508, 520, 523, 524] payload_rows_per_dst=[10838, 10950, 10846, 10962, 11098, 10896, 10992, 10990] dup_rows_per_dst=[5364, 5316, 5538, 5404, 5554, 5212, 5586, 5526] combine_rows_per_src=[10838, 10950, 10846, 10962, 11098, 10896, 10992, 10990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627621888 dispatch_meta_bytes=457832 combine_payload_bytes=627621888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628079720 ingress_bytes=703119360 link_bytes=703119360 token_bytes=1255701608 weight_movement_bytes=75497472 total_nvlink_bytes=1331199080 weight_to_token_ratio=0.060124 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916543200e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626622464 dispatch_meta_bytes=459072 combine_payload_bytes=626622464 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627081536 ingress_bytes=777617408 link_bytes=777617408 token_bytes=1253704000 weight_movement_bytes=150994944 total_nvlink_bytes=1404698944 weight_to_token_ratio=0.120439 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861283313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626786304 dispatch_meta_bytes=458256 combine_payload_bytes=626786304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627244560 ingress_bytes=702283776 link_bytes=702283776 token_bytes=1254030864 weight_movement_bytes=75497472 total_nvlink_bytes=1329528336 weight_to_token_ratio=0.060204 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905946819e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626475008 dispatch_meta_bytes=457680 combine_payload_bytes=626475008 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626932688 ingress_bytes=852967424 link_bytes=852967424 token_bytes=1253407696 weight_movement_bytes=226492416 total_nvlink_bytes=1479900112 weight_to_token_ratio=0.180701 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081682758e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87468 payload_rows_remote=76488 dup_rows=43604 dup_rows_remote=38082 dedup_ratio=0.332672 router_tokens_per_expert=[513, 519, 530, 511, 477, 471, 507, 524, 512, 501, 539, 495, 542, 519, 513, 511, 499, 529, 456, 540, 524, 524, 488, 554, 468, 524, 518, 496, 530, 507, 458, 534, 525, 525, 568, 519, 513, 493, 505, 544, 525, 485, 528, 485, 492, 544, 537, 515, 524, 536, 524, 520, 558, 511, 513, 536, 505, 527, 479, 523, 483, 528, 530, 531, 557, 476, 513, 536, 505, 523, 491, 478, 486, 501, 508, 484, 523, 495, 523, 534, 495, 491, 500, 518, 528, 455, 551, 522, 483, 522, 502, 495, 549, 513, 525, 510, 508, 497, 519, 526, 510, 490, 523, 538, 507, 529, 490, 565, 493, 491, 497, 463, 505, 530, 519, 518, 510, 499, 505, 494, 488, 494, 487, 498, 568, 476, 504, 539] payload_rows_per_dst=[10980, 10946, 10978, 11046, 10782, 10918, 10924, 10894] dup_rows_per_dst=[5522, 5422, 5558, 5656, 5304, 5400, 5368, 5374] combine_rows_per_src=[10980, 10946, 10978, 11046, 10782, 10918, 10924, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76642 dup_rows=43558 dup_rows_remote=38150 dedup_ratio=0.332321 router_tokens_per_expert=[529, 499, 501, 489, 517, 566, 528, 492, 516, 469, 458, 511, 521, 516, 515, 548, 545, 509, 499, 508, 535, 523, 515, 481, 492, 498, 518, 491, 528, 557, 492, 465, 482, 501, 482, 506, 479, 524, 539, 542, 506, 511, 529, 500, 536, 505, 504, 494, 495, 499, 546, 476, 546, 517, 508, 532, 522, 524, 490, 521, 498, 508, 471, 536, 508, 460, 513, 546, 502, 537, 519, 468, 552, 498, 495, 514, 500, 499, 530, 555, 521, 467, 495, 534, 526, 512, 491, 519, 519, 527, 479, 511, 501, 534, 507, 536, 505, 503, 508, 538, 503, 515, 505, 526, 518, 529, 531, 545, 524, 514, 479, 543, 525, 470, 510, 522, 480, 511, 529, 518, 531, 516, 577, 466, 509, 532, 504, 515] payload_rows_per_dst=[10898, 10962, 10872, 10894, 10920, 10924, 10994, 11050] dup_rows_per_dst=[5452, 5350, 5408, 5484, 5472, 5434, 5578, 5380] combine_rows_per_src=[10898, 10962, 10872, 10894, 10920, 10924, 10994, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87540 payload_rows_remote=76434 dup_rows=43532 dup_rows_remote=37980 dedup_ratio=0.332123 router_tokens_per_expert=[521, 519, 515, 544, 477, 508, 502, 526, 492, 483, 539, 478, 516, 508, 488, 508, 521, 498, 487, 553, 544, 524, 530, 509, 494, 516, 482, 517, 479, 507, 491, 479, 499, 504, 495, 482, 489, 524, 520, 521, 500, 528, 544, 495, 501, 501, 516, 518, 497, 498, 516, 512, 547, 514, 507, 505, 517, 491, 494, 526, 533, 509, 466, 518, 534, 508, 518, 516, 515, 546, 499, 528, 505, 504, 540, 503, 524, 520, 543, 502, 506, 519, 548, 518, 505, 517, 504, 536, 524, 489, 549, 497, 514, 542, 504, 557, 493, 479, 458, 500, 513, 506, 531, 508, 573, 503, 526, 493, 505, 520, 489, 499, 516, 502, 489, 501, 522, 542, 525, 517, 556, 515, 508, 507, 537, 510, 526, 491] payload_rows_per_dst=[10884, 10918, 10862, 10950, 10998, 11106, 10794, 11028] dup_rows_per_dst=[5364, 5344, 5412, 5350, 5612, 5552, 5398, 5500] combine_rows_per_src=[10884, 10918, 10862, 10950, 10998, 11106, 10794, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87294 payload_rows_remote=76392 dup_rows=43778 dup_rows_remote=38232 dedup_ratio=0.334000 router_tokens_per_expert=[536, 499, 523, 477, 529, 458, 485, 537, 562, 537, 523, 486, 517, 495, 535, 491, 555, 511, 510, 526, 524, 541, 535, 489, 470, 468, 512, 528, 490, 560, 517, 488, 468, 525, 471, 536, 504, 522, 482, 516, 507, 500, 477, 522, 498, 499, 508, 523, 507, 537, 522, 500, 469, 526, 521, 533, 508, 540, 547, 509, 523, 487, 533, 543, 484, 495, 523, 522, 570, 500, 499, 540, 496, 515, 504, 513, 541, 508, 516, 491, 484, 504, 537, 514, 496, 547, 518, 530, 529, 512, 480, 547, 547, 506, 481, 550, 501, 482, 533, 510, 515, 511, 520, 491, 538, 463, 539, 544, 519, 498, 493, 491, 476, 530, 520, 508, 515, 509, 528, 518, 544, 461, 511, 485, 505, 497, 486, 519] payload_rows_per_dst=[10930, 10902, 10806, 11026, 10898, 10982, 10922, 10828] dup_rows_per_dst=[5450, 5546, 5310, 5584, 5536, 5582, 5374, 5396] combine_rows_per_src=[10930, 10902, 10806, 11026, 10898, 10982, 10922, 10828] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76396 dup_rows=43566 dup_rows_remote=38000 dedup_ratio=0.332382 router_tokens_per_expert=[492, 499, 481, 496, 519, 530, 502, 489, 514, 514, 485, 528, 517, 496, 503, 507, 527, 515, 531, 566, 481, 512, 513, 499, 463, 493, 533, 481, 529, 512, 537, 495, 486, 549, 508, 523, 505, 510, 500, 530, 498, 502, 540, 524, 515, 534, 517, 533, 539, 510, 515, 528, 518, 524, 538, 467, 517, 494, 525, 507, 500, 525, 503, 515, 479, 467, 508, 527, 487, 545, 528, 518, 470, 451, 521, 514, 508, 550, 503, 525, 521, 525, 493, 501, 489, 525, 572, 552, 503, 475, 477, 503, 490, 492, 468, 505, 511, 521, 521, 537, 512, 528, 480, 500, 549, 497, 524, 514, 497, 520, 528, 509, 557, 497, 548, 499, 506, 520, 482, 533, 526, 536, 539, 530, 509, 537, 508, 511] payload_rows_per_dst=[10884, 11050, 10932, 10718, 11028, 10884, 10900, 11110] dup_rows_per_dst=[5260, 5324, 5616, 5096, 5810, 5298, 5596, 5566] combine_rows_per_src=[10884, 11050, 10932, 10718, 11028, 10884, 10900, 11110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76438 dup_rows=43598 dup_rows_remote=38296 dedup_ratio=0.332626 router_tokens_per_expert=[492, 506, 511, 525, 510, 529, 515, 533, 518, 500, 499, 481, 478, 496, 523, 474, 529, 488, 482, 541, 481, 522, 522, 529, 565, 489, 521, 499, 500, 462, 489, 509, 491, 548, 534, 492, 542, 508, 502, 523, 498, 533, 546, 535, 496, 491, 474, 506, 490, 526, 521, 513, 515, 502, 504, 530, 506, 516, 534, 506, 510, 503, 505, 488, 504, 509, 544, 511, 501, 527, 510, 465, 473, 549, 497, 555, 471, 494, 480, 499, 530, 529, 507, 535, 544, 512, 516, 527, 536, 497, 520, 534, 490, 535, 500, 538, 512, 554, 542, 522, 560, 500, 529, 502, 530, 549, 510, 542, 556, 485, 474, 530, 548, 493, 497, 493, 518, 504, 477, 521, 505, 529, 497, 540, 502, 507, 473, 490] payload_rows_per_dst=[10824, 10850, 10924, 11036, 10862, 11048, 11088, 10842] dup_rows_per_dst=[5356, 5406, 5514, 5302, 5316, 5652, 5706, 5346] combine_rows_per_src=[10824, 10850, 10924, 11036, 10862, 11048, 11088, 10842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87426 payload_rows_remote=76520 dup_rows=43646 dup_rows_remote=38308 dedup_ratio=0.332993 router_tokens_per_expert=[514, 510, 528, 516, 560, 504, 508, 520, 459, 497, 485, 517, 512, 469, 532, 529, 507, 512, 508, 505, 515, 526, 536, 489, 515, 486, 517, 465, 462, 536, 545, 493, 548, 499, 554, 476, 491, 506, 541, 540, 491, 465, 506, 526, 526, 559, 506, 534, 498, 459, 548, 523, 500, 499, 505, 493, 510, 557, 520, 505, 498, 522, 506, 510, 513, 495, 511, 524, 510, 533, 501, 515, 532, 507, 535, 502, 519, 527, 495, 500, 497, 543, 532, 519, 517, 520, 517, 490, 513, 496, 507, 505, 485, 544, 487, 547, 490, 510, 501, 462, 515, 501, 476, 505, 526, 498, 532, 502, 521, 547, 497, 539, 514, 491, 504, 504, 510, 503, 515, 515, 531, 539, 522, 525, 505, 521, 539, 540] payload_rows_per_dst=[10896, 10838, 11050, 10884, 10950, 10896, 10906, 11006] dup_rows_per_dst=[5424, 5396, 5486, 5422, 5488, 5542, 5338, 5550] combine_rows_per_src=[10896, 10838, 11050, 10884, 10950, 10896, 10906, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626589696 dispatch_meta_bytes=458280 combine_payload_bytes=626589696 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627047976 ingress_bytes=702087168 link_bytes=702087168 token_bytes=1253637672 weight_movement_bytes=75497472 total_nvlink_bytes=1329135144 weight_to_token_ratio=0.060223 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903453553e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627851264 dispatch_meta_bytes=459168 combine_payload_bytes=627851264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628310432 ingress_bytes=703348736 link_bytes=703348736 token_bytes=1256161696 weight_movement_bytes=75497472 total_nvlink_bytes=1331659168 weight_to_token_ratio=0.060102 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919452011e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76422 dup_rows=43738 dup_rows_remote=38324 dedup_ratio=0.333694 router_tokens_per_expert=[516, 483, 469, 527, 473, 518, 486, 551, 515, 521, 541, 516, 532, 497, 502, 502, 512, 538, 498, 514, 507, 481, 531, 502, 486, 506, 525, 560, 507, 508, 509, 524, 507, 533, 465, 490, 506, 547, 525, 515, 496, 500, 494, 482, 517, 491, 512, 551, 495, 497, 544, 519, 550, 515, 483, 502, 537, 488, 533, 489, 535, 489, 473, 519, 530, 500, 509, 475, 510, 512, 513, 517, 515, 528, 532, 482, 468, 516, 521, 535, 493, 542, 521, 555, 502, 558, 480, 538, 519, 504, 512, 536, 530, 471, 498, 516, 529, 530, 479, 472, 525, 522, 535, 543, 476, 530, 508, 553, 541, 509, 527, 475, 533, 494, 483, 525, 513, 525, 504, 494, 506, 509, 479, 546, 495, 546, 511, 525] payload_rows_per_dst=[10784, 10948, 10868, 10954, 10912, 10984, 10910, 10974] dup_rows_per_dst=[5514, 5468, 5394, 5382, 5414, 5566, 5598, 5402] combine_rows_per_src=[10784, 10948, 10868, 10954, 10912, 10984, 10910, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626147328 dispatch_meta_bytes=457656 combine_payload_bytes=626147328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626604984 ingress_bytes=701644800 link_bytes=701644800 token_bytes=1252752312 weight_movement_bytes=75497472 total_nvlink_bytes=1328249784 weight_to_token_ratio=0.060265 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897843704e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625803264 dispatch_meta_bytes=458496 combine_payload_bytes=625803264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626261760 ingress_bytes=701300736 link_bytes=701300736 token_bytes=1252065024 weight_movement_bytes=75497472 total_nvlink_bytes=1327562496 weight_to_token_ratio=0.060298 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893480488e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=457584 combine_payload_bytes=625836032 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626293616 ingress_bytes=852328448 link_bytes=852328448 token_bytes=1252129648 weight_movement_bytes=226492416 total_nvlink_bytes=1478622064 weight_to_token_ratio=0.180886 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080872446e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626180096 dispatch_meta_bytes=458936 combine_payload_bytes=626180096 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626639032 ingress_bytes=777175040 link_bytes=777175040 token_bytes=1252819128 weight_movement_bytes=150994944 total_nvlink_bytes=1403814072 weight_to_token_ratio=0.120524 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.855673464e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=459312 combine_payload_bytes=626851840 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627311152 ingress_bytes=853344256 link_bytes=853344256 token_bytes=1254162992 weight_movement_bytes=226492416 total_nvlink_bytes=1480655408 weight_to_token_ratio=0.180592 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082160634e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626049024 dispatch_meta_bytes=458984 combine_payload_bytes=626049024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626508008 ingress_bytes=701546496 link_bytes=701546496 token_bytes=1252557032 weight_movement_bytes=75497472 total_nvlink_bytes=1328054504 weight_to_token_ratio=0.060275 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.896597071e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87882 payload_rows_remote=76814 dup_rows=43190 dup_rows_remote=37736 dedup_ratio=0.329514 router_tokens_per_expert=[490, 538, 489, 523, 495, 520, 496, 485, 496, 521, 542, 515, 509, 523, 524, 476, 521, 498, 519, 517, 559, 483, 484, 506, 503, 499, 519, 536, 514, 526, 546, 492, 514, 510, 513, 510, 527, 512, 545, 500, 506, 519, 533, 489, 492, 517, 555, 519, 555, 530, 509, 490, 463, 471, 487, 544, 509, 527, 476, 537, 483, 530, 474, 534, 505, 492, 544, 505, 526, 522, 521, 502, 520, 509, 550, 497, 481, 550, 524, 496, 544, 511, 483, 534, 469, 494, 523, 520, 500, 482, 505, 536, 521, 574, 464, 527, 507, 512, 503, 504, 543, 539, 505, 531, 499, 527, 508, 496, 550, 491, 500, 476, 517, 518, 533, 500, 498, 506, 500, 499, 516, 499, 515, 522, 541, 482, 513, 511] payload_rows_per_dst=[11016, 11088, 11068, 10926, 10944, 10986, 10942, 10912] dup_rows_per_dst=[5268, 5356, 5454, 5312, 5544, 5388, 5440, 5428] combine_rows_per_src=[11016, 11088, 11068, 10926, 10944, 10986, 10942, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87546 payload_rows_remote=76340 dup_rows=43526 dup_rows_remote=37782 dedup_ratio=0.332077 router_tokens_per_expert=[490, 512, 540, 509, 542, 486, 505, 520, 495, 534, 483, 496, 488, 499, 551, 515, 526, 548, 528, 475, 510, 503, 513, 481, 505, 553, 521, 516, 521, 515, 522, 494, 530, 485, 508, 538, 502, 532, 503, 508, 473, 558, 523, 524, 496, 519, 510, 517, 517, 510, 518, 519, 537, 567, 485, 535, 525, 533, 531, 506, 542, 512, 487, 557, 501, 477, 527, 503, 553, 482, 500, 457, 539, 498, 520, 504, 536, 538, 517, 498, 506, 551, 537, 471, 518, 518, 506, 488, 485, 504, 513, 464, 523, 493, 499, 496, 475, 542, 482, 513, 517, 502, 463, 495, 520, 474, 527, 486, 560, 516, 501, 513, 503, 530, 522, 529, 525, 542, 522, 521, 489, 508, 476, 521, 534, 492, 499, 512] payload_rows_per_dst=[11206, 11064, 11018, 11108, 10862, 10726, 10702, 10860] dup_rows_per_dst=[5744, 5574, 5542, 5780, 5438, 4956, 5230, 5262] combine_rows_per_src=[11206, 11064, 11018, 11108, 10862, 10726, 10702, 10860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76574 dup_rows=43554 dup_rows_remote=38204 dedup_ratio=0.332291 router_tokens_per_expert=[523, 527, 521, 520, 514, 482, 525, 506, 497, 506, 493, 511, 502, 488, 520, 489, 518, 529, 515, 522, 512, 485, 506, 503, 500, 497, 513, 535, 497, 498, 497, 520, 537, 527, 517, 501, 518, 540, 557, 524, 495, 499, 484, 532, 478, 476, 486, 502, 485, 485, 512, 493, 480, 525, 541, 511, 547, 556, 503, 551, 536, 530, 508, 498, 498, 506, 552, 506, 485, 505, 496, 542, 495, 540, 507, 549, 531, 517, 514, 498, 500, 511, 500, 494, 511, 488, 512, 481, 550, 512, 509, 531, 521, 504, 500, 513, 489, 469, 511, 539, 510, 518, 523, 523, 520, 525, 522, 492, 569, 484, 502, 497, 519, 518, 515, 573, 439, 520, 508, 518, 500, 519, 506, 470, 577, 555, 499, 524] payload_rows_per_dst=[10802, 10944, 10854, 10944, 11080, 10836, 11008, 11050] dup_rows_per_dst=[5446, 5350, 5492, 5578, 5402, 5438, 5378, 5470] combine_rows_per_src=[10802, 10944, 10854, 10944, 11080, 10836, 11008, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87674 payload_rows_remote=76722 dup_rows=43398 dup_rows_remote=37940 dedup_ratio=0.331100 router_tokens_per_expert=[522, 492, 483, 509, 506, 540, 534, 527, 522, 510, 505, 497, 547, 485, 494, 532, 526, 528, 542, 532, 509, 514, 499, 513, 522, 528, 497, 492, 529, 487, 536, 498, 493, 537, 525, 449, 515, 525, 522, 512, 557, 533, 550, 504, 490, 499, 517, 505, 492, 507, 528, 544, 515, 449, 474, 491, 476, 509, 552, 488, 494, 519, 543, 524, 506, 490, 537, 501, 534, 487, 513, 521, 543, 527, 491, 512, 526, 494, 488, 543, 518, 517, 507, 529, 550, 491, 516, 535, 484, 484, 518, 522, 508, 515, 489, 546, 463, 498, 511, 523, 557, 475, 466, 548, 515, 518, 491, 508, 529, 497, 506, 495, 517, 476, 485, 504, 501, 488, 502, 527, 529, 533, 543, 492, 494, 555, 528, 525] payload_rows_per_dst=[10676, 11342, 11028, 10858, 10974, 10944, 10900, 10952] dup_rows_per_dst=[5014, 5882, 5426, 5276, 5452, 5460, 5430, 5458] combine_rows_per_src=[10676, 11342, 11028, 10858, 10974, 10944, 10900, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87578 payload_rows_remote=76564 dup_rows=43494 dup_rows_remote=37910 dedup_ratio=0.331833 router_tokens_per_expert=[484, 547, 461, 547, 533, 528, 515, 487, 520, 483, 515, 488, 496, 536, 531, 554, 523, 497, 559, 541, 474, 489, 514, 500, 499, 518, 522, 517, 493, 492, 539, 520, 494, 517, 519, 488, 491, 486, 526, 521, 495, 537, 518, 512, 474, 521, 542, 525, 520, 494, 545, 491, 529, 506, 498, 505, 540, 513, 512, 483, 543, 503, 503, 516, 508, 471, 462, 530, 516, 459, 506, 534, 489, 519, 494, 520, 534, 494, 529, 518, 530, 512, 557, 531, 511, 552, 508, 514, 541, 488, 514, 487, 522, 534, 471, 527, 524, 549, 512, 504, 522, 503, 527, 480, 499, 496, 517, 496, 523, 484, 504, 475, 513, 488, 536, 489, 528, 544, 513, 490, 536, 539, 527, 503, 508, 533, 500, 503] payload_rows_per_dst=[10946, 10958, 10944, 10986, 10872, 11014, 10924, 10934] dup_rows_per_dst=[5504, 5436, 5388, 5416, 5294, 5584, 5306, 5566] combine_rows_per_src=[10946, 10958, 10944, 10986, 10872, 11014, 10924, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87788 payload_rows_remote=76706 dup_rows=43284 dup_rows_remote=38052 dedup_ratio=0.330231 router_tokens_per_expert=[525, 537, 480, 523, 519, 549, 490, 504, 516, 537, 526, 505, 503, 506, 489, 494, 542, 487, 522, 511, 531, 484, 527, 520, 530, 533, 480, 522, 527, 476, 517, 489, 459, 516, 503, 507, 501, 489, 526, 537, 537, 464, 527, 501, 516, 511, 512, 516, 565, 534, 520, 477, 488, 518, 493, 490, 543, 505, 504, 495, 512, 512, 486, 515, 522, 495, 511, 511, 505, 526, 497, 532, 513, 528, 495, 511, 531, 476, 498, 494, 537, 496, 493, 502, 497, 563, 489, 537, 506, 516, 525, 492, 554, 461, 515, 549, 502, 524, 494, 540, 493, 514, 542, 507, 537, 531, 530, 507, 501, 495, 535, 489, 497, 511, 515, 551, 486, 544, 514, 503, 513, 520, 481, 512, 556, 533, 477, 525] payload_rows_per_dst=[10956, 10976, 10920, 11082, 10906, 11040, 10994, 10914] dup_rows_per_dst=[5450, 5420, 5324, 5232, 5384, 5424, 5488, 5562] combine_rows_per_src=[10956, 10976, 10920, 11082, 10906, 11040, 10994, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87604 payload_rows_remote=76740 dup_rows=43468 dup_rows_remote=38066 dedup_ratio=0.331635 router_tokens_per_expert=[502, 525, 492, 506, 536, 494, 530, 504, 499, 520, 494, 514, 542, 522, 536, 490, 514, 501, 506, 525, 506, 526, 544, 498, 507, 525, 504, 562, 513, 531, 499, 531, 523, 523, 498, 497, 484, 501, 516, 501, 500, 529, 520, 501, 486, 521, 517, 530, 486, 527, 532, 504, 523, 522, 514, 492, 495, 566, 558, 505, 517, 495, 511, 529, 534, 480, 519, 541, 524, 470, 504, 520, 524, 543, 526, 474, 507, 511, 479, 515, 534, 517, 483, 492, 523, 535, 531, 504, 489, 530, 503, 496, 500, 510, 517, 506, 543, 520, 477, 507, 508, 492, 507, 530, 531, 446, 502, 570, 509, 485, 518, 488, 530, 493, 506, 553, 514, 509, 440, 493, 528, 511, 555, 522, 492, 504, 464, 527] payload_rows_per_dst=[10960, 11080, 10972, 11020, 10910, 10920, 10864, 10878] dup_rows_per_dst=[5452, 5504, 5322, 5532, 5432, 5420, 5402, 5404] combine_rows_per_src=[10960, 11080, 10972, 11020, 10910, 10920, 10864, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=629260288 dispatch_meta_bytes=458200 combine_payload_bytes=629260288 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=629718488 ingress_bytes=704757760 link_bytes=704757760 token_bytes=1258978776 weight_movement_bytes=75497472 total_nvlink_bytes=1334476248 weight_to_token_ratio=0.059967 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.937320418e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625377280 dispatch_meta_bytes=456488 combine_payload_bytes=625377280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625833768 ingress_bytes=700874752 link_bytes=700874752 token_bytes=1251211048 weight_movement_bytes=75497472 total_nvlink_bytes=1326708520 weight_to_token_ratio=0.060340 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888078411e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627294208 dispatch_meta_bytes=459112 combine_payload_bytes=627294208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627753320 ingress_bytes=702791680 link_bytes=702791680 token_bytes=1255047528 weight_movement_bytes=75497472 total_nvlink_bytes=1330545000 weight_to_token_ratio=0.060155 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912387756e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87388 payload_rows_remote=76620 dup_rows=43684 dup_rows_remote=38062 dedup_ratio=0.333282 router_tokens_per_expert=[515, 501, 543, 476, 496, 532, 526, 539, 489, 514, 511, 548, 538, 509, 548, 492, 520, 475, 516, 528, 533, 491, 504, 472, 501, 474, 503, 495, 475, 464, 477, 502, 528, 530, 492, 471, 505, 503, 469, 528, 533, 518, 523, 526, 553, 531, 535, 490, 526, 548, 490, 557, 522, 509, 514, 524, 483, 535, 508, 498, 513, 518, 525, 519, 514, 544, 537, 520, 513, 518, 496, 505, 526, 478, 498, 498, 497, 499, 545, 507, 538, 534, 521, 532, 488, 498, 522, 575, 498, 520, 482, 531, 478, 504, 532, 494, 470, 518, 509, 524, 509, 513, 546, 531, 507, 484, 509, 520, 519, 491, 504, 535, 514, 514, 501, 536, 524, 515, 519, 512, 529, 492, 487, 473, 526, 499, 526, 507] payload_rows_per_dst=[11100, 10664, 10990, 10990, 10768, 10978, 10942, 10956] dup_rows_per_dst=[5454, 5196, 5480, 5588, 5622, 5516, 5436, 5392] combine_rows_per_src=[11100, 10664, 10990, 10990, 10768, 10978, 10942, 10956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628506624 dispatch_meta_bytes=458648 combine_payload_bytes=628506624 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628965272 ingress_bytes=854999040 link_bytes=854999040 token_bytes=1257471896 weight_movement_bytes=226492416 total_nvlink_bytes=1483964312 weight_to_token_ratio=0.180117 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084259133e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627212288 dispatch_meta_bytes=457896 combine_payload_bytes=627212288 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627670184 ingress_bytes=778207232 link_bytes=778207232 token_bytes=1254882472 weight_movement_bytes=150994944 total_nvlink_bytes=1405877416 weight_to_token_ratio=0.120326 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.868763111e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628375552 dispatch_meta_bytes=459032 combine_payload_bytes=628375552 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628834584 ingress_bytes=854867968 link_bytes=854867968 token_bytes=1257210136 weight_movement_bytes=226492416 total_nvlink_bytes=1483702552 weight_to_token_ratio=0.180155 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084092915e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628654080 dispatch_meta_bytes=459224 combine_payload_bytes=628654080 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629113304 ingress_bytes=704151552 link_bytes=704151552 token_bytes=1257767384 weight_movement_bytes=75497472 total_nvlink_bytes=1333264856 weight_to_token_ratio=0.060025 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.929632848e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627671040 dispatch_meta_bytes=458728 combine_payload_bytes=627671040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628129768 ingress_bytes=703168512 link_bytes=703168512 token_bytes=1255800808 weight_movement_bytes=75497472 total_nvlink_bytes=1331298280 weight_to_token_ratio=0.060119 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917166517e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76616 dup_rows=43554 dup_rows_remote=37890 dedup_ratio=0.332291 router_tokens_per_expert=[503, 513, 489, 520, 520, 506, 485, 524, 516, 498, 503, 486, 524, 503, 481, 501, 502, 529, 534, 490, 532, 509, 498, 489, 491, 503, 504, 555, 527, 506, 503, 548, 509, 496, 522, 517, 521, 525, 531, 525, 523, 524, 509, 534, 504, 492, 517, 534, 535, 482, 510, 545, 540, 520, 507, 565, 521, 517, 518, 502, 448, 494, 554, 473, 487, 546, 517, 472, 552, 524, 539, 542, 562, 531, 513, 528, 473, 497, 474, 459, 516, 538, 520, 520, 505, 498, 516, 506, 522, 544, 510, 517, 556, 514, 497, 484, 504, 508, 511, 508, 510, 490, 478, 506, 496, 484, 491, 500, 520, 525, 513, 493, 492, 500, 508, 523, 549, 540, 498, 519, 511, 509, 490, 528, 484, 511, 528, 524] payload_rows_per_dst=[10898, 10968, 10902, 11012, 10960, 10962, 10836, 10980] dup_rows_per_dst=[5246, 5472, 5664, 5450, 5472, 5564, 5238, 5448] combine_rows_per_src=[10898, 10968, 10902, 11012, 10960, 10962, 10836, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87404 payload_rows_remote=76396 dup_rows=43668 dup_rows_remote=37946 dedup_ratio=0.333160 router_tokens_per_expert=[549, 520, 496, 494, 517, 496, 533, 520, 479, 536, 505, 505, 498, 548, 532, 566, 510, 518, 509, 544, 525, 476, 533, 532, 456, 522, 520, 522, 478, 478, 467, 491, 448, 527, 508, 497, 493, 540, 485, 524, 541, 502, 542, 544, 566, 470, 499, 465, 514, 547, 551, 529, 505, 496, 532, 516, 525, 511, 524, 529, 534, 501, 491, 513, 541, 495, 538, 491, 506, 498, 502, 519, 508, 464, 505, 495, 505, 502, 523, 572, 471, 527, 528, 466, 534, 471, 480, 500, 512, 512, 510, 511, 527, 543, 499, 518, 520, 508, 482, 522, 492, 487, 494, 525, 504, 510, 511, 534, 505, 531, 536, 510, 520, 497, 533, 532, 469, 526, 499, 539, 501, 506, 540, 535, 514, 532, 496, 509] payload_rows_per_dst=[11008, 10876, 10926, 10976, 11106, 10786, 10872, 10854] dup_rows_per_dst=[5722, 5286, 5376, 5546, 5506, 5432, 5470, 5330] combine_rows_per_src=[11008, 10876, 10926, 10976, 11106, 10786, 10872, 10854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87630 payload_rows_remote=76578 dup_rows=43442 dup_rows_remote=37994 dedup_ratio=0.331436 router_tokens_per_expert=[510, 555, 502, 506, 509, 500, 494, 524, 477, 485, 507, 484, 502, 503, 566, 516, 504, 518, 494, 497, 536, 522, 512, 518, 506, 533, 534, 527, 481, 507, 539, 522, 568, 524, 506, 502, 508, 498, 564, 520, 534, 510, 491, 516, 523, 501, 508, 519, 511, 526, 507, 497, 490, 483, 521, 524, 500, 520, 507, 520, 503, 494, 500, 492, 513, 503, 520, 487, 519, 485, 456, 484, 487, 530, 501, 514, 511, 536, 520, 503, 537, 544, 468, 534, 517, 539, 506, 498, 558, 534, 519, 480, 509, 510, 530, 488, 542, 486, 484, 502, 545, 499, 509, 495, 548, 497, 544, 541, 531, 494, 499, 493, 540, 483, 477, 523, 534, 504, 525, 463, 528, 496, 511, 504, 530, 539, 523, 530] payload_rows_per_dst=[10972, 11052, 11016, 10844, 10808, 11008, 10970, 10960] dup_rows_per_dst=[5308, 5448, 5568, 5346, 5330, 5534, 5448, 5460] combine_rows_per_src=[10972, 11052, 11016, 10844, 10808, 11008, 10970, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76498 dup_rows=43654 dup_rows_remote=38100 dedup_ratio=0.333054 router_tokens_per_expert=[516, 530, 570, 521, 484, 515, 500, 499, 525, 546, 491, 506, 496, 524, 525, 494, 476, 510, 502, 506, 543, 510, 504, 480, 487, 545, 508, 514, 486, 503, 536, 522, 529, 525, 525, 502, 541, 534, 513, 544, 489, 529, 494, 490, 490, 532, 526, 564, 495, 513, 478, 463, 472, 513, 499, 501, 542, 513, 463, 543, 500, 494, 511, 496, 510, 526, 500, 543, 511, 542, 497, 506, 477, 581, 487, 527, 481, 503, 529, 519, 539, 500, 527, 474, 520, 498, 537, 528, 532, 485, 497, 515, 513, 493, 513, 478, 527, 514, 485, 556, 488, 525, 533, 491, 505, 526, 520, 529, 529, 461, 508, 517, 520, 510, 543, 476, 496, 482, 521, 540, 530, 509, 529, 548, 536, 505, 500, 492] payload_rows_per_dst=[10984, 10934, 10648, 10748, 10960, 10986, 11238, 10920] dup_rows_per_dst=[5500, 5330, 4988, 5244, 5518, 5480, 6040, 5554] combine_rows_per_src=[10984, 10934, 10648, 10748, 10960, 10986, 11238, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87512 payload_rows_remote=76632 dup_rows=43560 dup_rows_remote=38188 dedup_ratio=0.332336 router_tokens_per_expert=[512, 437, 542, 491, 505, 483, 554, 463, 498, 524, 531, 515, 522, 498, 527, 504, 492, 480, 478, 526, 514, 499, 536, 542, 527, 539, 544, 502, 528, 527, 489, 508, 500, 497, 514, 522, 511, 470, 541, 513, 483, 509, 512, 493, 534, 552, 536, 496, 523, 541, 505, 523, 509, 520, 487, 539, 493, 456, 492, 515, 523, 513, 474, 513, 507, 504, 505, 477, 529, 444, 478, 493, 490, 509, 540, 560, 506, 516, 519, 521, 535, 511, 526, 491, 512, 517, 520, 506, 515, 502, 508, 532, 525, 498, 536, 536, 487, 526, 503, 493, 555, 494, 534, 532, 501, 488, 570, 479, 551, 534, 497, 496, 557, 522, 496, 517, 520, 543, 540, 510, 545, 504, 505, 490, 506, 506, 513, 508] payload_rows_per_dst=[10908, 11034, 10782, 10880, 10814, 11066, 11024, 11004] dup_rows_per_dst=[5304, 5428, 5584, 5372, 5382, 5474, 5456, 5560] combine_rows_per_src=[10908, 11034, 10782, 10880, 10814, 11066, 11024, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87300 payload_rows_remote=76272 dup_rows=43772 dup_rows_remote=38296 dedup_ratio=0.333954 router_tokens_per_expert=[536, 551, 507, 509, 483, 533, 535, 521, 515, 537, 555, 508, 510, 497, 528, 549, 471, 497, 542, 498, 520, 507, 509, 483, 493, 519, 472, 544, 519, 492, 468, 520, 505, 550, 503, 527, 523, 523, 532, 489, 522, 524, 505, 490, 492, 510, 503, 522, 527, 506, 509, 519, 471, 499, 474, 537, 514, 486, 471, 518, 504, 518, 533, 520, 485, 505, 497, 483, 530, 486, 504, 570, 508, 500, 481, 477, 478, 502, 527, 488, 474, 530, 528, 544, 495, 567, 522, 521, 511, 482, 521, 500, 488, 498, 509, 562, 511, 475, 509, 535, 513, 568, 567, 465, 473, 492, 508, 535, 526, 518, 551, 507, 526, 472, 551, 503, 485, 511, 514, 557, 504, 517, 513, 524, 509, 546, 499, 525] payload_rows_per_dst=[11122, 10866, 10950, 10798, 10804, 11028, 10824, 10908] dup_rows_per_dst=[5626, 5242, 5490, 5414, 5238, 5476, 5682, 5604] combine_rows_per_src=[11122, 10866, 10950, 10798, 10804, 11028, 10824, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627638272 dispatch_meta_bytes=458024 combine_payload_bytes=627638272 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628096296 ingress_bytes=854130688 link_bytes=854130688 token_bytes=1255734568 weight_movement_bytes=226492416 total_nvlink_bytes=1482226984 weight_to_token_ratio=0.180366 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083157940e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=457368 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626293400 ingress_bytes=776830976 link_bytes=776830976 token_bytes=1252129432 weight_movement_bytes=150994944 total_nvlink_bytes=1403124376 weight_to_token_ratio=0.120591 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851310248e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87662 payload_rows_remote=76658 dup_rows=43410 dup_rows_remote=37978 dedup_ratio=0.331192 router_tokens_per_expert=[529, 525, 530, 519, 483, 512, 502, 497, 506, 484, 525, 485, 509, 481, 493, 561, 558, 509, 492, 486, 520, 489, 465, 500, 520, 515, 500, 509, 492, 560, 511, 501, 526, 471, 509, 501, 498, 531, 517, 501, 505, 541, 516, 472, 523, 503, 509, 563, 512, 488, 530, 475, 481, 528, 529, 521, 497, 511, 497, 520, 499, 494, 525, 486, 500, 520, 511, 490, 547, 535, 472, 513, 524, 537, 515, 563, 501, 550, 469, 522, 492, 526, 526, 560, 524, 496, 523, 497, 563, 497, 505, 516, 486, 512, 528, 505, 518, 491, 547, 523, 485, 491, 526, 519, 491, 545, 529, 498, 508, 515, 536, 496, 498, 513, 527, 480, 462, 555, 539, 516, 543, 531, 514, 496, 534, 566, 479, 493] payload_rows_per_dst=[11002, 10866, 10872, 10886, 11066, 10914, 11004, 11052] dup_rows_per_dst=[5280, 5388, 5500, 5300, 5472, 5598, 5432, 5440] combine_rows_per_src=[11002, 10866, 10872, 10886, 11066, 10914, 11004, 11052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=458288 combine_payload_bytes=627326976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627785264 ingress_bytes=702824448 link_bytes=702824448 token_bytes=1255112240 weight_movement_bytes=75497472 total_nvlink_bytes=1330609712 weight_to_token_ratio=0.060152 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912803301e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76592 dup_rows=43528 dup_rows_remote=37914 dedup_ratio=0.332092 router_tokens_per_expert=[505, 534, 503, 509, 548, 498, 493, 507, 548, 493, 502, 539, 505, 495, 523, 552, 496, 527, 549, 524, 533, 511, 465, 479, 493, 495, 517, 513, 478, 523, 500, 530, 509, 533, 500, 541, 515, 524, 514, 482, 497, 530, 541, 542, 498, 473, 515, 525, 514, 508, 508, 501, 539, 549, 519, 492, 520, 543, 502, 516, 508, 489, 540, 490, 491, 460, 491, 533, 532, 515, 530, 513, 539, 520, 513, 509, 518, 558, 566, 495, 482, 528, 497, 544, 516, 523, 541, 472, 523, 507, 538, 487, 523, 490, 466, 511, 524, 511, 539, 499, 504, 486, 511, 508, 523, 430, 517, 494, 524, 472, 499, 497, 527, 500, 565, 501, 511, 479, 543, 503, 514, 513, 535, 517, 496, 507, 511, 481] payload_rows_per_dst=[10984, 10930, 10978, 11014, 10952, 10888, 10882, 10916] dup_rows_per_dst=[5524, 5336, 5500, 5462, 5614, 5408, 5194, 5490] combine_rows_per_src=[10984, 10930, 10978, 11014, 10952, 10888, 10882, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626671616 dispatch_meta_bytes=458392 combine_payload_bytes=626671616 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627130008 ingress_bytes=853164032 link_bytes=853164032 token_bytes=1253801624 weight_movement_bytes=226492416 total_nvlink_bytes=1480294040 weight_to_token_ratio=0.180645 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081932084e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=459280 combine_payload_bytes=627769344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628228624 ingress_bytes=703266816 link_bytes=703266816 token_bytes=1255997968 weight_movement_bytes=75497472 total_nvlink_bytes=1331495440 weight_to_token_ratio=0.060110 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918413150e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624820224 dispatch_meta_bytes=458272 combine_payload_bytes=624820224 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625278496 ingress_bytes=775815168 link_bytes=775815168 token_bytes=1250098720 weight_movement_bytes=150994944 total_nvlink_bytes=1401093664 weight_to_token_ratio=0.120786 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.838428373e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627982336 dispatch_meta_bytes=458544 combine_payload_bytes=627982336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628440880 ingress_bytes=703479808 link_bytes=703479808 token_bytes=1256423216 weight_movement_bytes=75497472 total_nvlink_bytes=1331920688 weight_to_token_ratio=0.060089 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921114188e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627441664 dispatch_meta_bytes=458024 combine_payload_bytes=627441664 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627899688 ingress_bytes=778436608 link_bytes=778436608 token_bytes=1255341352 weight_movement_bytes=150994944 total_nvlink_bytes=1406336296 weight_to_token_ratio=0.120282 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.871671922e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76504 dup_rows=43674 dup_rows_remote=38214 dedup_ratio=0.333206 router_tokens_per_expert=[511, 494, 509, 516, 525, 523, 504, 518, 527, 525, 481, 523, 547, 541, 512, 509, 485, 527, 533, 538, 531, 511, 509, 519, 497, 530, 530, 502, 529, 531, 495, 480, 531, 530, 506, 508, 472, 558, 492, 522, 526, 481, 491, 542, 517, 493, 509, 499, 512, 531, 469, 526, 551, 483, 515, 506, 489, 505, 508, 485, 502, 494, 483, 523, 533, 515, 489, 464, 529, 517, 502, 513, 535, 547, 538, 514, 504, 498, 507, 513, 501, 535, 528, 446, 543, 539, 517, 563, 484, 539, 513, 510, 512, 540, 499, 506, 502, 533, 483, 509, 525, 546, 535, 498, 493, 538, 515, 498, 508, 463, 486, 495, 478, 506, 491, 540, 492, 485, 491, 576, 509, 521, 509, 504, 516, 487, 514, 526] payload_rows_per_dst=[11038, 10986, 10894, 10856, 10930, 11006, 10848, 10840] dup_rows_per_dst=[5492, 5508, 5460, 5308, 5506, 5544, 5406, 5450] combine_rows_per_src=[11038, 10986, 10894, 10856, 10930, 11006, 10848, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87332 payload_rows_remote=76196 dup_rows=43740 dup_rows_remote=37978 dedup_ratio=0.333710 router_tokens_per_expert=[483, 473, 520, 508, 475, 521, 522, 502, 540, 457, 486, 513, 511, 530, 523, 494, 519, 501, 523, 532, 508, 522, 513, 528, 490, 511, 513, 489, 527, 526, 496, 524, 542, 535, 525, 494, 476, 526, 484, 505, 548, 530, 499, 486, 499, 552, 518, 524, 537, 486, 534, 518, 507, 522, 482, 506, 523, 488, 532, 560, 535, 499, 534, 516, 522, 554, 508, 514, 545, 495, 494, 448, 495, 553, 543, 521, 494, 506, 535, 553, 524, 527, 561, 482, 497, 535, 531, 521, 508, 544, 520, 508, 521, 512, 502, 506, 521, 516, 498, 551, 535, 484, 471, 531, 452, 501, 502, 485, 482, 473, 499, 487, 527, 523, 499, 499, 479, 530, 530, 523, 526, 505, 507, 503, 526, 518, 466, 506] payload_rows_per_dst=[11136, 11022, 10922, 11194, 10848, 10758, 10742, 10710] dup_rows_per_dst=[5762, 5602, 5584, 6040, 5194, 5164, 5234, 5160] combine_rows_per_src=[11136, 11022, 10922, 11194, 10848, 10758, 10742, 10710] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87354 payload_rows_remote=76394 dup_rows=43718 dup_rows_remote=38452 dedup_ratio=0.333542 router_tokens_per_expert=[486, 521, 518, 552, 475, 539, 534, 503, 508, 518, 543, 516, 507, 514, 504, 465, 497, 528, 511, 487, 493, 499, 477, 525, 512, 571, 534, 493, 517, 461, 496, 512, 528, 505, 518, 480, 515, 538, 509, 532, 547, 515, 489, 567, 544, 500, 499, 530, 516, 480, 525, 471, 527, 513, 534, 482, 524, 503, 514, 504, 555, 535, 487, 537, 531, 495, 547, 485, 505, 514, 527, 531, 505, 515, 522, 489, 492, 506, 530, 497, 503, 509, 502, 515, 513, 526, 482, 517, 519, 493, 496, 544, 490, 513, 513, 496, 557, 531, 476, 492, 513, 513, 523, 521, 548, 495, 510, 540, 524, 489, 493, 507, 484, 508, 486, 488, 511, 509, 500, 551, 538, 493, 495, 521, 510, 518, 506, 525] payload_rows_per_dst=[10962, 10960, 10916, 10928, 10926, 10850, 10892, 10920] dup_rows_per_dst=[5444, 5266, 5716, 5486, 5456, 5412, 5572, 5366] combine_rows_per_src=[10962, 10960, 10916, 10928, 10926, 10850, 10892, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87678 payload_rows_remote=76654 dup_rows=43394 dup_rows_remote=37836 dedup_ratio=0.331070 router_tokens_per_expert=[509, 564, 508, 499, 502, 510, 508, 484, 512, 524, 508, 491, 526, 468, 518, 536, 512, 509, 504, 488, 512, 506, 509, 528, 472, 487, 520, 508, 511, 484, 504, 511, 470, 493, 478, 509, 524, 514, 573, 528, 495, 528, 497, 506, 535, 486, 536, 493, 510, 509, 504, 496, 527, 520, 543, 537, 480, 535, 501, 520, 491, 523, 531, 539, 521, 554, 484, 503, 456, 521, 560, 540, 532, 436, 518, 497, 504, 548, 500, 511, 523, 507, 522, 539, 507, 504, 522, 517, 506, 514, 555, 529, 531, 492, 505, 526, 508, 522, 505, 554, 503, 542, 515, 506, 490, 512, 523, 474, 517, 518, 536, 494, 520, 509, 482, 483, 537, 502, 528, 496, 493, 507, 473, 487, 522, 538, 572, 521] payload_rows_per_dst=[11022, 10710, 10790, 11020, 10992, 11004, 11116, 11024] dup_rows_per_dst=[5312, 5080, 5210, 5612, 5378, 5620, 5624, 5558] combine_rows_per_src=[11022, 10710, 10790, 11020, 10992, 11004, 11116, 11024] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76464 dup_rows=43598 dup_rows_remote=38088 dedup_ratio=0.332626 router_tokens_per_expert=[494, 510, 485, 501, 500, 490, 481, 502, 480, 508, 548, 510, 478, 506, 509, 505, 525, 477, 539, 507, 550, 487, 531, 493, 511, 530, 511, 500, 511, 503, 560, 512, 530, 473, 526, 520, 525, 478, 504, 529, 554, 458, 451, 532, 553, 491, 502, 522, 522, 540, 495, 517, 509, 515, 484, 492, 523, 513, 537, 523, 533, 487, 540, 530, 470, 530, 504, 536, 534, 504, 476, 517, 477, 543, 501, 453, 508, 507, 472, 477, 519, 510, 548, 514, 533, 521, 526, 510, 520, 518, 501, 538, 512, 519, 530, 522, 503, 516, 524, 512, 492, 506, 538, 533, 518, 503, 494, 517, 505, 528, 537, 514, 509, 516, 505, 534, 470, 527, 507, 527, 548, 519, 520, 525, 516, 520, 527, 514] payload_rows_per_dst=[10752, 10942, 10892, 11010, 10840, 10990, 10952, 11096] dup_rows_per_dst=[5262, 5552, 5404, 5510, 5178, 5692, 5528, 5472] combine_rows_per_src=[10752, 10942, 10892, 11010, 10840, 10990, 10952, 11096] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=458872 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627179640 ingress_bytes=853213184 link_bytes=853213184 token_bytes=1253900408 weight_movement_bytes=226492416 total_nvlink_bytes=1480392824 weight_to_token_ratio=0.180630 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081994416e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87394 payload_rows_remote=76468 dup_rows=43678 dup_rows_remote=38336 dedup_ratio=0.333237 router_tokens_per_expert=[426, 512, 519, 540, 534, 529, 496, 475, 483, 522, 494, 533, 523, 511, 514, 521, 506, 481, 503, 513, 518, 494, 522, 520, 506, 511, 537, 540, 514, 484, 494, 488, 497, 517, 511, 514, 517, 525, 467, 530, 548, 510, 534, 498, 505, 550, 555, 504, 497, 484, 545, 471, 538, 496, 528, 517, 530, 489, 503, 487, 498, 500, 505, 552, 485, 509, 503, 525, 525, 528, 542, 557, 507, 511, 541, 512, 504, 530, 539, 523, 539, 520, 527, 517, 496, 530, 520, 481, 477, 490, 500, 529, 486, 498, 519, 505, 547, 526, 497, 506, 493, 548, 529, 513, 484, 508, 459, 561, 488, 520, 445, 543, 530, 493, 507, 536, 522, 514, 509, 523, 495, 488, 477, 523, 533, 512, 520, 527] payload_rows_per_dst=[10848, 10892, 10934, 10934, 11078, 10926, 10942, 10840] dup_rows_per_dst=[5416, 5370, 5630, 5346, 5604, 5342, 5392, 5578] combine_rows_per_src=[10848, 10892, 10934, 10934, 11078, 10926, 10942, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76570 dup_rows=43582 dup_rows_remote=38112 dedup_ratio=0.332504 router_tokens_per_expert=[500, 517, 530, 482, 509, 519, 485, 550, 476, 512, 470, 492, 495, 550, 569, 506, 509, 518, 534, 531, 482, 496, 489, 506, 522, 552, 531, 513, 523, 551, 540, 494, 515, 546, 506, 507, 532, 553, 519, 528, 451, 531, 526, 527, 508, 500, 523, 517, 492, 534, 509, 503, 509, 525, 516, 487, 495, 474, 484, 529, 504, 505, 488, 459, 494, 526, 515, 513, 526, 484, 497, 540, 495, 497, 516, 476, 474, 499, 504, 548, 523, 487, 518, 528, 483, 512, 512, 485, 524, 514, 555, 550, 533, 536, 491, 517, 532, 488, 500, 486, 533, 504, 494, 531, 481, 496, 500, 513, 528, 560, 533, 516, 530, 555, 491, 472, 503, 536, 481, 518, 516, 512, 499, 499, 518, 536, 504, 544] payload_rows_per_dst=[10952, 11056, 11002, 10774, 10848, 10990, 10920, 10948] dup_rows_per_dst=[5372, 5526, 5576, 5252, 5360, 5546, 5470, 5480] combine_rows_per_src=[10952, 11056, 11002, 10774, 10848, 10990, 10920, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=624197632 dispatch_meta_bytes=456696 combine_payload_bytes=624197632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624654328 ingress_bytes=699695104 link_bytes=699695104 token_bytes=1248851960 weight_movement_bytes=75497472 total_nvlink_bytes=1324349432 weight_to_token_ratio=0.060454 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.873118814e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87690 payload_rows_remote=76768 dup_rows=43382 dup_rows_remote=37884 dedup_ratio=0.330978 router_tokens_per_expert=[528, 499, 549, 517, 488, 465, 550, 529, 491, 528, 470, 480, 533, 527, 523, 506, 474, 519, 530, 513, 519, 558, 476, 513, 536, 521, 522, 529, 532, 512, 499, 499, 502, 492, 513, 533, 485, 544, 446, 494, 473, 524, 547, 523, 502, 524, 563, 497, 527, 502, 507, 521, 507, 537, 497, 519, 532, 508, 550, 494, 477, 527, 524, 522, 482, 510, 532, 511, 530, 514, 520, 534, 528, 514, 491, 491, 494, 512, 532, 515, 520, 489, 503, 539, 505, 562, 493, 526, 514, 515, 500, 532, 533, 446, 547, 526, 501, 532, 496, 510, 481, 496, 519, 519, 485, 516, 515, 501, 513, 490, 476, 513, 494, 496, 520, 501, 511, 517, 514, 500, 497, 483, 535, 556, 518, 510, 522, 491] payload_rows_per_dst=[11038, 10996, 10940, 11018, 10922, 10986, 10848, 10942] dup_rows_per_dst=[5328, 5508, 5384, 5484, 5498, 5514, 5278, 5388] combine_rows_per_src=[11038, 10996, 10940, 11018, 10922, 10986, 10848, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625819648 dispatch_meta_bytes=459384 combine_payload_bytes=625819648 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626279032 ingress_bytes=852312064 link_bytes=852312064 token_bytes=1252098680 weight_movement_bytes=226492416 total_nvlink_bytes=1478591096 weight_to_token_ratio=0.180890 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080851669e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627949568 dispatch_meta_bytes=457960 combine_payload_bytes=627949568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628407528 ingress_bytes=703447040 link_bytes=703447040 token_bytes=1256357096 weight_movement_bytes=75497472 total_nvlink_bytes=1331854568 weight_to_token_ratio=0.060092 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920698644e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626393088 dispatch_meta_bytes=458208 combine_payload_bytes=626393088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626851296 ingress_bytes=701890560 link_bytes=701890560 token_bytes=1253244384 weight_movement_bytes=75497472 total_nvlink_bytes=1328741856 weight_to_token_ratio=0.060242 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900960286e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=459216 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626885072 ingress_bytes=777420800 link_bytes=777420800 token_bytes=1253310928 weight_movement_bytes=150994944 total_nvlink_bytes=1404305872 weight_to_token_ratio=0.120477 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.858790047e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627261440 dispatch_meta_bytes=458728 combine_payload_bytes=627261440 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627720168 ingress_bytes=702758912 link_bytes=702758912 token_bytes=1254981608 weight_movement_bytes=75497472 total_nvlink_bytes=1330479080 weight_to_token_ratio=0.060158 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911972212e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628883456 dispatch_meta_bytes=458608 combine_payload_bytes=628883456 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629342064 ingress_bytes=704380928 link_bytes=704380928 token_bytes=1258225520 weight_movement_bytes=75497472 total_nvlink_bytes=1333722992 weight_to_token_ratio=0.060003 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.932541658e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87560 payload_rows_remote=76470 dup_rows=43512 dup_rows_remote=37928 dedup_ratio=0.331970 router_tokens_per_expert=[485, 535, 509, 482, 478, 467, 503, 494, 500, 509, 522, 521, 483, 509, 506, 508, 505, 524, 518, 535, 494, 519, 520, 466, 501, 516, 510, 540, 518, 521, 516, 521, 509, 512, 501, 504, 508, 545, 525, 532, 520, 512, 535, 521, 554, 519, 503, 537, 485, 533, 530, 519, 477, 512, 538, 532, 496, 514, 494, 546, 490, 564, 487, 541, 533, 516, 496, 490, 541, 519, 523, 522, 509, 519, 514, 517, 530, 487, 506, 496, 492, 543, 538, 532, 500, 520, 493, 538, 557, 492, 531, 517, 521, 498, 496, 495, 505, 476, 485, 500, 535, 493, 521, 514, 492, 502, 519, 514, 480, 531, 506, 516, 514, 531, 493, 476, 508, 503, 523, 515, 518, 462, 534, 551, 503, 492, 517, 496] payload_rows_per_dst=[10822, 11006, 11090, 11010, 10906, 10950, 10894, 10882] dup_rows_per_dst=[5200, 5442, 5584, 5506, 5530, 5576, 5284, 5390] combine_rows_per_src=[10822, 11006, 11090, 11010, 10906, 10950, 10894, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87630 payload_rows_remote=76528 dup_rows=43442 dup_rows_remote=37664 dedup_ratio=0.331436 router_tokens_per_expert=[545, 500, 512, 507, 503, 515, 503, 506, 528, 508, 526, 529, 506, 498, 492, 504, 521, 504, 488, 551, 469, 494, 503, 556, 481, 509, 530, 502, 525, 510, 514, 537, 528, 506, 584, 522, 514, 518, 513, 527, 527, 523, 493, 491, 469, 533, 493, 510, 494, 481, 506, 486, 505, 519, 510, 512, 515, 507, 570, 524, 483, 536, 511, 530, 524, 520, 525, 527, 531, 526, 487, 512, 498, 508, 536, 526, 484, 548, 512, 531, 515, 507, 540, 501, 466, 507, 523, 564, 510, 493, 463, 521, 497, 513, 512, 524, 546, 487, 514, 516, 483, 513, 497, 489, 497, 495, 469, 505, 535, 539, 500, 535, 522, 517, 474, 515, 504, 541, 503, 461, 536, 501, 457, 528, 494, 528, 551, 517] payload_rows_per_dst=[11102, 11074, 10928, 11042, 11228, 10406, 10938, 10912] dup_rows_per_dst=[5778, 5314, 5574, 5336, 5882, 4870, 5302, 5386] combine_rows_per_src=[11102, 11074, 10928, 11042, 11228, 10406, 10938, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87702 payload_rows_remote=76676 dup_rows=43370 dup_rows_remote=37798 dedup_ratio=0.330887 router_tokens_per_expert=[508, 503, 492, 512, 517, 504, 531, 500, 529, 541, 502, 518, 511, 509, 519, 485, 519, 509, 494, 489, 506, 508, 557, 502, 514, 514, 538, 561, 495, 494, 514, 550, 527, 509, 502, 524, 518, 510, 486, 516, 538, 509, 504, 544, 520, 502, 515, 521, 517, 525, 505, 514, 490, 533, 526, 500, 504, 473, 455, 520, 522, 520, 519, 560, 510, 465, 508, 535, 516, 497, 560, 524, 487, 507, 555, 509, 478, 545, 465, 518, 510, 536, 514, 513, 473, 494, 488, 537, 518, 461, 515, 539, 538, 484, 534, 527, 483, 512, 500, 531, 497, 480, 509, 512, 531, 543, 554, 513, 505, 508, 524, 467, 545, 480, 504, 551, 507, 470, 520, 495, 502, 512, 492, 521, 511, 514, 482, 528] payload_rows_per_dst=[10992, 10772, 11008, 11066, 11004, 10968, 10866, 11026] dup_rows_per_dst=[5370, 5274, 5482, 5782, 5354, 5394, 5142, 5572] combine_rows_per_src=[10992, 10772, 11008, 11066, 11004, 10968, 10866, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76472 dup_rows=43654 dup_rows_remote=38102 dedup_ratio=0.333054 router_tokens_per_expert=[515, 547, 511, 533, 521, 550, 535, 497, 464, 484, 485, 492, 520, 510, 535, 524, 540, 525, 512, 481, 493, 505, 479, 501, 510, 521, 532, 553, 537, 497, 514, 549, 473, 516, 485, 469, 454, 505, 506, 487, 533, 523, 535, 521, 510, 518, 479, 496, 522, 518, 483, 491, 550, 511, 531, 497, 528, 511, 535, 497, 510, 518, 469, 517, 497, 502, 536, 506, 488, 499, 477, 485, 524, 517, 532, 503, 549, 465, 549, 506, 551, 560, 539, 547, 484, 556, 533, 478, 535, 497, 525, 495, 551, 520, 529, 503, 498, 537, 503, 493, 532, 502, 496, 544, 514, 522, 505, 494, 528, 502, 477, 498, 516, 493, 530, 499, 499, 530, 541, 536, 510, 475, 493, 547, 488, 503, 510, 513] payload_rows_per_dst=[10964, 10946, 10766, 10990, 10872, 11056, 10908, 10916] dup_rows_per_dst=[5482, 5552, 5254, 5386, 5398, 5750, 5382, 5450] combine_rows_per_src=[10964, 10946, 10766, 10990, 10872, 11056, 10908, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626442240 dispatch_meta_bytes=457592 combine_payload_bytes=626442240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626899832 ingress_bytes=701939712 link_bytes=701939712 token_bytes=1253342072 weight_movement_bytes=75497472 total_nvlink_bytes=1328839544 weight_to_token_ratio=0.060237 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901583603e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87336 payload_rows_remote=76408 dup_rows=43736 dup_rows_remote=38504 dedup_ratio=0.333679 router_tokens_per_expert=[518, 483, 498, 505, 501, 509, 524, 493, 478, 508, 533, 510, 505, 524, 526, 480, 492, 499, 517, 511, 521, 504, 543, 491, 560, 498, 499, 529, 502, 531, 520, 513, 550, 512, 486, 490, 495, 512, 494, 510, 519, 523, 536, 529, 510, 508, 529, 536, 503, 502, 526, 508, 493, 508, 511, 474, 536, 539, 514, 503, 437, 530, 496, 500, 482, 532, 508, 457, 535, 504, 530, 532, 498, 521, 510, 533, 494, 527, 520, 513, 479, 514, 510, 499, 503, 513, 547, 550, 524, 505, 500, 543, 541, 505, 515, 501, 555, 535, 521, 556, 514, 511, 537, 482, 498, 529, 493, 534, 521, 507, 499, 520, 525, 499, 493, 529, 524, 503, 527, 508, 523, 486, 492, 498, 487, 485, 529, 527] payload_rows_per_dst=[10762, 10924, 10960, 10928, 10916, 10994, 10968, 10884] dup_rows_per_dst=[5428, 5536, 5518, 5232, 5476, 5504, 5656, 5386] combine_rows_per_src=[10762, 10924, 10960, 10928, 10916, 10994, 10968, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87682 payload_rows_remote=76512 dup_rows=43390 dup_rows_remote=37918 dedup_ratio=0.331039 router_tokens_per_expert=[519, 544, 475, 489, 520, 518, 544, 502, 488, 510, 472, 503, 503, 502, 472, 548, 524, 527, 520, 536, 508, 489, 484, 540, 515, 526, 522, 499, 523, 528, 524, 532, 514, 541, 512, 537, 503, 540, 491, 538, 501, 504, 475, 471, 551, 500, 497, 518, 536, 501, 498, 506, 514, 559, 513, 494, 519, 503, 511, 472, 477, 566, 510, 539, 535, 497, 511, 505, 495, 496, 459, 504, 523, 509, 530, 515, 483, 538, 461, 503, 491, 500, 534, 518, 479, 536, 564, 521, 508, 531, 487, 515, 528, 545, 526, 538, 496, 551, 495, 547, 546, 467, 508, 497, 525, 505, 532, 490, 540, 501, 508, 481, 530, 500, 494, 509, 521, 554, 480, 524, 525, 481, 498, 499, 537, 471, 489, 533] payload_rows_per_dst=[10840, 11064, 10918, 10920, 10810, 11170, 11016, 10944] dup_rows_per_dst=[5378, 5530, 5468, 5516, 5318, 5472, 5362, 5346] combine_rows_per_src=[10840, 11064, 10918, 10920, 10810, 11170, 11016, 10944] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87836 payload_rows_remote=76742 dup_rows=43236 dup_rows_remote=37590 dedup_ratio=0.329865 router_tokens_per_expert=[535, 521, 501, 491, 479, 507, 538, 492, 523, 508, 535, 498, 508, 482, 522, 540, 515, 516, 476, 490, 500, 543, 501, 496, 536, 499, 498, 492, 484, 565, 520, 551, 529, 536, 481, 500, 538, 500, 512, 490, 507, 475, 480, 461, 512, 501, 533, 503, 505, 493, 504, 516, 486, 511, 485, 555, 513, 549, 479, 539, 531, 531, 516, 518, 498, 497, 534, 499, 472, 476, 506, 516, 486, 528, 512, 494, 496, 505, 510, 502, 560, 519, 519, 513, 524, 511, 526, 502, 551, 489, 519, 515, 534, 524, 518, 518, 487, 527, 567, 505, 531, 521, 506, 540, 538, 532, 538, 480, 521, 497, 544, 536, 547, 520, 476, 526, 515, 502, 436, 480, 515, 484, 503, 522, 533, 541, 513, 529] payload_rows_per_dst=[10958, 10908, 10852, 10930, 10924, 11192, 11094, 10978] dup_rows_per_dst=[5402, 5456, 5264, 5532, 5138, 5492, 5646, 5306] combine_rows_per_src=[10958, 10908, 10852, 10930, 10924, 11192, 11094, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=456768 combine_payload_bytes=626917376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627374144 ingress_bytes=702414848 link_bytes=702414848 token_bytes=1254291520 weight_movement_bytes=75497472 total_nvlink_bytes=1329788992 weight_to_token_ratio=0.060191 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907608996e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76532 dup_rows=43528 dup_rows_remote=38174 dedup_ratio=0.332092 router_tokens_per_expert=[536, 534, 516, 513, 514, 566, 525, 473, 493, 510, 468, 548, 497, 526, 552, 507, 484, 489, 498, 500, 523, 493, 548, 485, 535, 529, 491, 476, 475, 481, 549, 503, 543, 503, 498, 517, 523, 476, 540, 478, 534, 559, 512, 508, 470, 512, 511, 483, 541, 516, 488, 539, 517, 501, 455, 567, 474, 524, 483, 558, 476, 484, 509, 498, 531, 510, 502, 522, 490, 524, 528, 487, 496, 528, 516, 499, 502, 497, 513, 538, 539, 513, 495, 512, 524, 522, 495, 509, 527, 540, 503, 530, 517, 517, 520, 520, 500, 489, 544, 527, 482, 487, 515, 542, 500, 518, 474, 525, 531, 543, 499, 505, 550, 501, 484, 524, 525, 532, 546, 515, 510, 500, 521, 535, 492, 509, 483, 528] payload_rows_per_dst=[11006, 10858, 10912, 10872, 11012, 10978, 10946, 10960] dup_rows_per_dst=[5550, 5260, 5422, 5388, 5354, 5588, 5416, 5550] combine_rows_per_src=[11006, 10858, 10912, 10872, 11012, 10978, 10946, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=457896 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628587688 ingress_bytes=703627264 link_bytes=703627264 token_bytes=1256717480 weight_movement_bytes=75497472 total_nvlink_bytes=1332214952 weight_to_token_ratio=0.060075 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922984138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626458624 dispatch_meta_bytes=458296 combine_payload_bytes=626458624 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626916920 ingress_bytes=777453568 link_bytes=777453568 token_bytes=1253375544 weight_movement_bytes=150994944 total_nvlink_bytes=1404370488 weight_to_token_ratio=0.120471 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.859205591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=459648 combine_payload_bytes=625934336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626393984 ingress_bytes=701431808 link_bytes=701431808 token_bytes=1252328320 weight_movement_bytes=75497472 total_nvlink_bytes=1327825792 weight_to_token_ratio=0.060286 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895142665e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626786304 dispatch_meta_bytes=457720 combine_payload_bytes=626786304 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627244024 ingress_bytes=853278720 link_bytes=853278720 token_bytes=1254030328 weight_movement_bytes=226492416 total_nvlink_bytes=1480522744 weight_to_token_ratio=0.180612 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082077525e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628670464 dispatch_meta_bytes=457328 combine_payload_bytes=628670464 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=629127792 ingress_bytes=855162880 link_bytes=855162880 token_bytes=1257798256 weight_movement_bytes=226492416 total_nvlink_bytes=1484290672 weight_to_token_ratio=0.180071 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084466905e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626950144 dispatch_meta_bytes=458824 combine_payload_bytes=626950144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627408968 ingress_bytes=702447616 link_bytes=702447616 token_bytes=1254359112 weight_movement_bytes=75497472 total_nvlink_bytes=1329856584 weight_to_token_ratio=0.060188 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908024541e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87764 payload_rows_remote=76830 dup_rows=43308 dup_rows_remote=37782 dedup_ratio=0.330414 router_tokens_per_expert=[519, 520, 502, 553, 486, 540, 505, 502, 519, 494, 510, 514, 525, 508, 545, 522, 493, 513, 527, 478, 506, 532, 534, 499, 523, 488, 489, 499, 502, 547, 522, 531, 522, 519, 508, 506, 481, 545, 495, 526, 515, 495, 526, 505, 496, 499, 552, 540, 506, 509, 528, 517, 579, 497, 491, 514, 558, 522, 504, 521, 484, 510, 530, 535, 525, 536, 528, 484, 513, 483, 514, 556, 518, 551, 468, 513, 552, 524, 516, 539, 490, 524, 509, 483, 475, 498, 473, 504, 487, 504, 489, 498, 495, 492, 527, 494, 514, 496, 499, 498, 506, 490, 506, 504, 551, 517, 505, 523, 525, 506, 516, 485, 498, 502, 525, 524, 536, 515, 516, 489, 486, 500, 516, 531, 496, 476, 519, 522] payload_rows_per_dst=[10986, 10962, 10934, 11056, 11052, 10772, 11020, 10982] dup_rows_per_dst=[5542, 5404, 5526, 5554, 5588, 5112, 5262, 5320] combine_rows_per_src=[10986, 10962, 10934, 11056, 11052, 10772, 11020, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87394 payload_rows_remote=76236 dup_rows=43678 dup_rows_remote=37884 dedup_ratio=0.333237 router_tokens_per_expert=[484, 562, 519, 497, 527, 510, 532, 517, 497, 487, 516, 517, 501, 496, 522, 516, 521, 549, 509, 488, 518, 498, 517, 490, 524, 551, 502, 497, 561, 491, 509, 539, 503, 490, 544, 510, 526, 466, 508, 529, 512, 531, 510, 503, 492, 500, 550, 531, 524, 519, 521, 517, 525, 489, 512, 479, 481, 493, 476, 498, 500, 536, 494, 493, 496, 484, 493, 526, 478, 462, 503, 541, 521, 512, 515, 508, 507, 512, 526, 475, 554, 492, 498, 542, 508, 532, 538, 536, 511, 501, 493, 562, 520, 506, 522, 506, 522, 484, 530, 505, 523, 506, 525, 518, 495, 518, 543, 493, 486, 533, 559, 515, 518, 481, 506, 566, 504, 509, 536, 495, 486, 487, 543, 475, 515, 521, 513, 520] payload_rows_per_dst=[11158, 11006, 11046, 10862, 11076, 10382, 11028, 10836] dup_rows_per_dst=[5794, 5566, 5616, 5252, 5610, 4934, 5392, 5514] combine_rows_per_src=[11158, 11006, 11046, 10862, 11076, 10382, 11028, 10836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76474 dup_rows=43648 dup_rows_remote=38112 dedup_ratio=0.333008 router_tokens_per_expert=[507, 474, 493, 520, 533, 502, 486, 510, 548, 521, 498, 525, 478, 442, 524, 531, 504, 514, 527, 524, 532, 525, 525, 494, 523, 520, 494, 539, 539, 511, 492, 520, 488, 536, 484, 535, 544, 495, 482, 559, 514, 497, 509, 505, 537, 526, 521, 504, 524, 488, 533, 517, 525, 510, 532, 497, 501, 499, 531, 546, 507, 525, 498, 501, 513, 473, 569, 519, 507, 508, 520, 496, 485, 490, 519, 526, 504, 533, 486, 491, 475, 506, 526, 523, 540, 521, 529, 521, 513, 476, 502, 497, 493, 502, 505, 480, 488, 509, 477, 508, 530, 503, 532, 526, 561, 477, 555, 475, 542, 531, 494, 566, 499, 525, 479, 528, 532, 515, 498, 484, 491, 536, 488, 524, 522, 495, 509, 544] payload_rows_per_dst=[10764, 10954, 10914, 10702, 10974, 11032, 11134, 10950] dup_rows_per_dst=[5184, 5612, 5558, 5046, 5376, 5686, 5650, 5536] combine_rows_per_src=[10764, 10954, 10914, 10702, 10974, 11032, 11134, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76548 dup_rows=43520 dup_rows_remote=38046 dedup_ratio=0.332031 router_tokens_per_expert=[500, 522, 517, 526, 509, 508, 514, 538, 521, 480, 506, 557, 489, 480, 521, 534, 506, 515, 525, 536, 463, 550, 517, 514, 525, 515, 537, 539, 499, 512, 500, 486, 491, 511, 514, 511, 487, 535, 469, 477, 470, 510, 504, 503, 549, 526, 533, 512, 530, 502, 509, 498, 526, 495, 521, 502, 523, 538, 530, 508, 526, 542, 538, 520, 521, 507, 517, 549, 516, 556, 529, 508, 496, 481, 502, 489, 505, 500, 511, 508, 487, 559, 520, 506, 509, 505, 539, 519, 507, 498, 510, 548, 467, 533, 546, 508, 500, 512, 496, 570, 540, 516, 482, 523, 454, 511, 520, 467, 482, 528, 497, 494, 504, 521, 481, 523, 465, 514, 512, 526, 477, 499, 509, 494, 548, 498, 546, 500] payload_rows_per_dst=[11028, 11004, 10818, 11004, 10960, 11040, 10822, 10876] dup_rows_per_dst=[5416, 5474, 5386, 5612, 5430, 5482, 5362, 5358] combine_rows_per_src=[11028, 11004, 10818, 11004, 10960, 11040, 10822, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629391360 dispatch_meta_bytes=458448 combine_payload_bytes=629391360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629849808 ingress_bytes=704888832 link_bytes=704888832 token_bytes=1259241168 weight_movement_bytes=75497472 total_nvlink_bytes=1334738640 weight_to_token_ratio=0.059955 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.938982596e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87736 payload_rows_remote=76736 dup_rows=43336 dup_rows_remote=37680 dedup_ratio=0.330627 router_tokens_per_expert=[498, 489, 500, 511, 530, 546, 542, 466, 509, 512, 549, 470, 530, 503, 514, 466, 493, 504, 520, 507, 482, 525, 480, 514, 563, 529, 526, 523, 537, 542, 535, 491, 537, 490, 469, 515, 448, 520, 518, 518, 493, 509, 542, 536, 536, 503, 503, 516, 497, 496, 466, 533, 578, 518, 539, 562, 501, 537, 513, 550, 500, 503, 506, 529, 475, 506, 511, 489, 527, 499, 518, 524, 530, 498, 504, 511, 519, 516, 526, 488, 516, 516, 513, 515, 523, 511, 519, 510, 541, 492, 490, 488, 539, 465, 449, 519, 509, 517, 507, 486, 469, 490, 506, 532, 532, 546, 555, 527, 486, 483, 504, 505, 524, 479, 551, 524, 516, 507, 507, 532, 493, 522, 527, 510, 512, 510, 501, 533] payload_rows_per_dst=[10882, 11030, 10820, 11000, 10920, 11002, 10988, 11094] dup_rows_per_dst=[5388, 5512, 5486, 5656, 5362, 5210, 5320, 5402] combine_rows_per_src=[10882, 11030, 10820, 11000, 10920, 11002, 10988, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87444 payload_rows_remote=76576 dup_rows=43628 dup_rows_remote=38182 dedup_ratio=0.332855 router_tokens_per_expert=[485, 519, 513, 485, 505, 536, 498, 524, 511, 498, 504, 509, 512, 495, 495, 503, 522, 551, 512, 504, 537, 480, 492, 536, 554, 519, 494, 493, 540, 553, 527, 469, 500, 516, 507, 515, 494, 538, 496, 493, 532, 532, 508, 521, 492, 478, 531, 496, 501, 516, 531, 554, 495, 496, 487, 526, 513, 502, 514, 491, 493, 513, 508, 528, 526, 549, 525, 526, 515, 519, 504, 521, 502, 519, 473, 519, 492, 534, 520, 507, 473, 509, 515, 512, 571, 518, 509, 492, 472, 481, 509, 528, 504, 506, 513, 545, 511, 496, 527, 536, 504, 506, 513, 521, 508, 529, 535, 504, 497, 515, 529, 494, 573, 576, 524, 529, 490, 501, 514, 514, 496, 518, 478, 523, 501, 487, 490, 497] payload_rows_per_dst=[10944, 11066, 10892, 10852, 10958, 10868, 10944, 10920] dup_rows_per_dst=[5240, 5500, 5406, 5484, 5544, 5446, 5506, 5502] combine_rows_per_src=[10944, 11066, 10892, 10852, 10958, 10868, 10944, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87768 payload_rows_remote=76750 dup_rows=43304 dup_rows_remote=37976 dedup_ratio=0.330383 router_tokens_per_expert=[514, 516, 487, 511, 524, 535, 494, 534, 525, 539, 531, 525, 526, 525, 521, 453, 540, 537, 559, 517, 518, 500, 511, 549, 528, 506, 491, 519, 508, 499, 515, 525, 495, 523, 561, 488, 503, 507, 510, 535, 521, 512, 481, 469, 456, 489, 530, 502, 494, 504, 474, 518, 520, 519, 512, 515, 539, 482, 512, 464, 527, 511, 531, 532, 516, 474, 495, 462, 524, 526, 520, 498, 510, 526, 488, 531, 529, 525, 506, 520, 520, 495, 531, 522, 528, 497, 558, 520, 506, 533, 489, 513, 536, 488, 545, 505, 538, 489, 553, 506, 519, 518, 493, 504, 488, 485, 499, 521, 490, 556, 504, 510, 498, 504, 517, 489, 499, 489, 468, 523, 485, 530, 536, 510, 530, 492, 541, 498] payload_rows_per_dst=[10968, 11018, 10898, 10920, 10896, 11034, 11018, 11016] dup_rows_per_dst=[5552, 5626, 5266, 5388, 5404, 5538, 5328, 5202] combine_rows_per_src=[10968, 11018, 10898, 10920, 10896, 11034, 11018, 11016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87642 payload_rows_remote=76730 dup_rows=43430 dup_rows_remote=37914 dedup_ratio=0.331345 router_tokens_per_expert=[521, 509, 503, 490, 490, 500, 510, 469, 518, 528, 525, 510, 546, 529, 496, 521, 470, 501, 499, 503, 536, 480, 498, 528, 534, 480, 451, 555, 456, 512, 521, 516, 485, 520, 524, 519, 488, 514, 510, 521, 503, 518, 510, 481, 508, 513, 558, 511, 525, 551, 449, 484, 540, 508, 527, 545, 474, 532, 489, 547, 511, 497, 512, 542, 482, 513, 539, 512, 542, 539, 544, 521, 519, 540, 475, 512, 462, 487, 523, 504, 537, 537, 540, 523, 498, 509, 518, 492, 533, 500, 540, 506, 477, 535, 505, 529, 498, 508, 497, 516, 524, 520, 549, 513, 504, 487, 505, 502, 485, 497, 517, 474, 520, 508, 526, 547, 505, 515, 548, 546, 533, 520, 517, 478, 526, 512, 520, 505] payload_rows_per_dst=[10908, 10828, 10902, 11042, 10912, 11172, 10878, 11000] dup_rows_per_dst=[5422, 5252, 5464, 5424, 5516, 5386, 5314, 5652] combine_rows_per_src=[10908, 10828, 10902, 11042, 10912, 11172, 10878, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624525312 dispatch_meta_bytes=456480 combine_payload_bytes=624525312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624981792 ingress_bytes=700022784 link_bytes=700022784 token_bytes=1249507104 weight_movement_bytes=75497472 total_nvlink_bytes=1325004576 weight_to_token_ratio=0.060422 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.877274258e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626475008 dispatch_meta_bytes=458344 combine_payload_bytes=626475008 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626933352 ingress_bytes=701972480 link_bytes=701972480 token_bytes=1253408360 weight_movement_bytes=75497472 total_nvlink_bytes=1328905832 weight_to_token_ratio=0.060234 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901999147e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627081216 dispatch_meta_bytes=458376 combine_payload_bytes=627081216 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627539592 ingress_bytes=702578688 link_bytes=702578688 token_bytes=1254620808 weight_movement_bytes=75497472 total_nvlink_bytes=1330118280 weight_to_token_ratio=0.060176 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909686718e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628621312 dispatch_meta_bytes=457664 combine_payload_bytes=628621312 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=629078976 ingress_bytes=1006108672 link_bytes=1006108672 token_bytes=1257700288 weight_movement_bytes=377487360 total_nvlink_bytes=1635187648 weight_to_token_ratio=0.300141 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.275887417e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627310592 dispatch_meta_bytes=459032 combine_payload_bytes=627310592 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627769624 ingress_bytes=929300480 link_bytes=929300480 token_bytes=1255080216 weight_movement_bytes=301989888 total_nvlink_bytes=1557070104 weight_to_token_ratio=0.240614 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178483818e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628736000 dispatch_meta_bytes=458904 combine_payload_bytes=628736000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629194904 ingress_bytes=704233472 link_bytes=704233472 token_bytes=1257930904 weight_movement_bytes=75497472 total_nvlink_bytes=1333428376 weight_to_token_ratio=0.060017 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930671709e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628572160 dispatch_meta_bytes=458576 combine_payload_bytes=628572160 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629030736 ingress_bytes=704069632 link_bytes=704069632 token_bytes=1257602896 weight_movement_bytes=75497472 total_nvlink_bytes=1333100368 weight_to_token_ratio=0.060033 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.928593987e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87626 payload_rows_remote=76716 dup_rows=43446 dup_rows_remote=37982 dedup_ratio=0.331467 router_tokens_per_expert=[521, 504, 515, 474, 503, 498, 527, 512, 543, 487, 540, 497, 499, 499, 537, 487, 511, 525, 495, 506, 488, 501, 540, 494, 481, 482, 515, 552, 530, 542, 483, 511, 499, 481, 494, 535, 522, 491, 493, 554, 506, 499, 513, 519, 508, 535, 502, 536, 540, 530, 513, 527, 527, 493, 535, 521, 520, 485, 500, 493, 521, 532, 476, 503, 541, 512, 503, 533, 527, 507, 465, 500, 541, 479, 523, 498, 502, 498, 510, 525, 481, 529, 493, 504, 522, 520, 495, 528, 527, 528, 490, 552, 546, 567, 462, 502, 494, 536, 484, 526, 533, 544, 503, 523, 496, 523, 521, 515, 526, 525, 486, 505, 494, 530, 542, 502, 512, 528, 515, 478, 502, 539, 526, 490, 491, 518, 493, 524] payload_rows_per_dst=[10978, 10886, 10910, 10932, 10872, 11042, 11052, 10954] dup_rows_per_dst=[5308, 5426, 5464, 5500, 5456, 5450, 5428, 5414] combine_rows_per_src=[10978, 10886, 10910, 10932, 10872, 11042, 11052, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87388 payload_rows_remote=76462 dup_rows=43684 dup_rows_remote=38166 dedup_ratio=0.333282 router_tokens_per_expert=[489, 552, 507, 490, 518, 522, 558, 487, 535, 505, 538, 464, 502, 551, 511, 493, 531, 471, 522, 527, 514, 489, 504, 465, 485, 516, 483, 534, 521, 456, 506, 536, 525, 526, 548, 508, 485, 532, 515, 523, 502, 499, 520, 502, 484, 517, 454, 501, 514, 525, 509, 506, 495, 535, 493, 501, 488, 486, 503, 546, 551, 510, 548, 544, 507, 552, 518, 525, 518, 486, 512, 512, 487, 524, 500, 521, 506, 513, 522, 536, 500, 511, 517, 496, 510, 499, 495, 505, 516, 508, 543, 490, 540, 539, 479, 543, 524, 513, 545, 510, 487, 536, 504, 513, 506, 501, 487, 532, 567, 537, 526, 520, 510, 506, 527, 486, 496, 543, 458, 528, 515, 522, 493, 477, 504, 515, 530, 511] payload_rows_per_dst=[10926, 10800, 10984, 10888, 10910, 11028, 11018, 10834] dup_rows_per_dst=[5518, 5442, 5468, 5328, 5568, 5354, 5598, 5408] combine_rows_per_src=[10926, 10800, 10984, 10888, 10910, 11028, 11018, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87412 payload_rows_remote=76408 dup_rows=43660 dup_rows_remote=37956 dedup_ratio=0.333099 router_tokens_per_expert=[507, 541, 515, 554, 506, 515, 520, 489, 575, 525, 468, 479, 544, 518, 521, 527, 514, 496, 550, 483, 507, 507, 518, 510, 494, 511, 495, 493, 481, 518, 504, 512, 506, 527, 491, 485, 553, 537, 488, 509, 494, 510, 506, 492, 531, 490, 477, 505, 496, 489, 496, 522, 502, 525, 511, 561, 507, 506, 478, 530, 531, 530, 522, 537, 525, 505, 518, 475, 538, 489, 546, 509, 502, 508, 483, 491, 502, 520, 544, 523, 518, 564, 533, 545, 508, 532, 535, 473, 518, 547, 546, 519, 511, 480, 513, 516, 511, 478, 493, 496, 454, 552, 491, 558, 491, 488, 480, 508, 540, 507, 476, 526, 523, 517, 526, 517, 517, 462, 532, 532, 535, 522, 498, 507, 481, 509, 485, 547] payload_rows_per_dst=[10868, 10940, 10854, 10948, 11044, 11078, 10676, 11004] dup_rows_per_dst=[5588, 5246, 5348, 5320, 5464, 5856, 5134, 5704] combine_rows_per_src=[10868, 10940, 10854, 10948, 11044, 11078, 10676, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628457472 dispatch_meta_bytes=458792 combine_payload_bytes=628457472 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628916264 ingress_bytes=703954944 link_bytes=703954944 token_bytes=1257373736 weight_movement_bytes=75497472 total_nvlink_bytes=1332871208 weight_to_token_ratio=0.060044 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927139581e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87564 payload_rows_remote=76642 dup_rows=43508 dup_rows_remote=38042 dedup_ratio=0.331940 router_tokens_per_expert=[527, 550, 476, 484, 503, 505, 503, 499, 497, 513, 498, 559, 532, 491, 498, 483, 489, 503, 555, 493, 497, 526, 527, 528, 529, 496, 528, 515, 545, 477, 499, 487, 513, 513, 540, 500, 551, 514, 516, 488, 461, 499, 495, 526, 499, 486, 523, 485, 502, 564, 503, 531, 523, 523, 527, 478, 512, 508, 546, 530, 499, 494, 512, 493, 522, 516, 491, 496, 502, 480, 508, 535, 543, 504, 502, 521, 496, 487, 559, 501, 527, 546, 500, 499, 504, 523, 480, 473, 527, 554, 486, 507, 509, 516, 535, 473, 541, 504, 528, 527, 525, 518, 497, 535, 494, 505, 521, 497, 530, 508, 503, 556, 513, 481, 505, 506, 559, 529, 508, 482, 541, 545, 528, 524, 492, 514, 511, 521] payload_rows_per_dst=[10916, 10922, 10850, 11058, 10874, 10880, 11080, 10984] dup_rows_per_dst=[5320, 5466, 5368, 5432, 5452, 5438, 5498, 5534] combine_rows_per_src=[10916, 10922, 10850, 11058, 10874, 10880, 11080, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76606 dup_rows=43566 dup_rows_remote=38040 dedup_ratio=0.332382 router_tokens_per_expert=[502, 524, 512, 471, 479, 503, 501, 492, 557, 529, 493, 566, 531, 534, 538, 482, 483, 545, 534, 545, 496, 543, 504, 546, 478, 530, 532, 486, 452, 499, 502, 480, 497, 516, 515, 514, 492, 511, 517, 478, 484, 517, 475, 490, 514, 473, 508, 533, 488, 492, 545, 506, 554, 492, 528, 521, 547, 494, 495, 538, 509, 539, 481, 484, 490, 504, 519, 537, 537, 491, 505, 460, 526, 562, 512, 557, 476, 472, 547, 501, 526, 511, 504, 525, 542, 502, 537, 519, 513, 549, 517, 505, 503, 566, 504, 489, 548, 533, 470, 512, 517, 482, 540, 475, 488, 550, 484, 500, 476, 480, 494, 568, 510, 490, 556, 506, 517, 540, 482, 510, 570, 533, 492, 511, 542, 520, 543, 473] payload_rows_per_dst=[10900, 10862, 10882, 10900, 10918, 11036, 10952, 11056] dup_rows_per_dst=[5528, 5448, 5186, 5526, 5474, 5588, 5282, 5534] combine_rows_per_src=[10900, 10862, 10882, 10900, 10918, 11036, 10952, 11056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87110 payload_rows_remote=76262 dup_rows=43962 dup_rows_remote=38434 dedup_ratio=0.335403 router_tokens_per_expert=[517, 527, 523, 489, 543, 485, 512, 459, 523, 484, 503, 480, 527, 486, 523, 550, 478, 526, 485, 542, 484, 524, 521, 532, 455, 537, 509, 485, 544, 524, 492, 500, 510, 507, 514, 506, 554, 533, 512, 493, 516, 542, 470, 511, 538, 482, 520, 499, 517, 525, 527, 555, 562, 542, 511, 522, 510, 553, 498, 507, 513, 538, 489, 516, 541, 526, 458, 510, 553, 516, 513, 504, 490, 520, 524, 515, 522, 512, 539, 523, 497, 505, 530, 524, 497, 492, 503, 536, 501, 511, 515, 491, 478, 496, 482, 512, 529, 546, 495, 528, 493, 528, 517, 523, 522, 502, 514, 493, 518, 509, 456, 515, 528, 505, 528, 499, 525, 493, 502, 522, 500, 478, 509, 543, 486, 480, 514, 539] payload_rows_per_dst=[10790, 10854, 10876, 11102, 10950, 10750, 10848, 10940] dup_rows_per_dst=[5472, 5422, 5538, 5668, 5582, 5390, 5528, 5362] combine_rows_per_src=[10790, 10854, 10876, 11102, 10950, 10750, 10848, 10940] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87660 payload_rows_remote=76840 dup_rows=43412 dup_rows_remote=38008 dedup_ratio=0.331207 router_tokens_per_expert=[552, 516, 580, 515, 516, 509, 506, 514, 477, 501, 496, 517, 553, 514, 519, 509, 523, 512, 515, 506, 509, 461, 486, 506, 482, 520, 504, 529, 477, 530, 473, 544, 529, 542, 496, 512, 522, 544, 515, 494, 525, 530, 519, 566, 514, 480, 519, 517, 547, 506, 505, 502, 487, 485, 516, 553, 500, 517, 520, 468, 514, 545, 542, 519, 482, 501, 501, 475, 524, 495, 475, 499, 517, 536, 513, 518, 514, 512, 515, 525, 487, 539, 486, 464, 523, 548, 500, 491, 533, 498, 530, 513, 478, 501, 512, 509, 515, 501, 529, 517, 506, 502, 538, 557, 510, 541, 493, 511, 486, 485, 495, 494, 508, 551, 495, 531, 541, 524, 488, 512, 473, 503, 546, 503, 518, 520, 493, 515] payload_rows_per_dst=[10976, 10832, 11088, 11082, 10944, 10820, 10966, 10952] dup_rows_per_dst=[5612, 5322, 5560, 5370, 5260, 5404, 5394, 5490] combine_rows_per_src=[10976, 10832, 11088, 11082, 10944, 10820, 10966, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87758 payload_rows_remote=76844 dup_rows=43314 dup_rows_remote=37872 dedup_ratio=0.330460 router_tokens_per_expert=[524, 520, 489, 468, 501, 509, 484, 517, 467, 503, 510, 501, 556, 503, 529, 502, 477, 522, 536, 519, 554, 512, 496, 516, 527, 510, 536, 507, 486, 508, 519, 495, 536, 517, 490, 503, 474, 514, 540, 469, 499, 504, 477, 511, 517, 494, 546, 499, 495, 486, 517, 511, 484, 501, 488, 508, 542, 526, 512, 512, 489, 446, 518, 481, 481, 520, 500, 513, 493, 575, 511, 509, 521, 491, 478, 520, 482, 512, 532, 540, 535, 468, 513, 499, 509, 533, 515, 522, 511, 550, 501, 510, 526, 517, 546, 522, 534, 520, 511, 542, 560, 551, 525, 541, 514, 482, 530, 512, 487, 503, 494, 551, 506, 545, 501, 521, 498, 492, 491, 511, 534, 538, 541, 532, 508, 516, 543, 538] payload_rows_per_dst=[10952, 10980, 10892, 10828, 10914, 11076, 11014, 11102] dup_rows_per_dst=[5214, 5460, 5288, 5204, 5442, 5478, 5700, 5528] combine_rows_per_src=[10952, 10980, 10892, 10828, 10914, 11076, 11014, 11102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626376704 dispatch_meta_bytes=458512 combine_payload_bytes=626376704 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626835216 ingress_bytes=852869120 link_bytes=852869120 token_bytes=1253211920 weight_movement_bytes=226492416 total_nvlink_bytes=1479704336 weight_to_token_ratio=0.180730 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081558095e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=457456 combine_payload_bytes=625934336 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626391792 ingress_bytes=776929280 link_bytes=776929280 token_bytes=1252326128 weight_movement_bytes=150994944 total_nvlink_bytes=1403321072 weight_to_token_ratio=0.120572 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.852556881e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627851264 dispatch_meta_bytes=458736 combine_payload_bytes=627851264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628310000 ingress_bytes=703348736 link_bytes=703348736 token_bytes=1256161264 weight_movement_bytes=75497472 total_nvlink_bytes=1331658736 weight_to_token_ratio=0.060102 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919452011e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627556352 dispatch_meta_bytes=458584 combine_payload_bytes=627556352 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628014936 ingress_bytes=854048768 link_bytes=854048768 token_bytes=1255571288 weight_movement_bytes=226492416 total_nvlink_bytes=1482063704 weight_to_token_ratio=0.180390 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083054054e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=624738304 dispatch_meta_bytes=458784 combine_payload_bytes=624738304 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=625197088 ingress_bytes=775733248 link_bytes=775733248 token_bytes=1249935392 weight_movement_bytes=150994944 total_nvlink_bytes=1400930336 weight_to_token_ratio=0.120802 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.837389512e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629473280 dispatch_meta_bytes=459392 combine_payload_bytes=629473280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629932672 ingress_bytes=704970752 link_bytes=704970752 token_bytes=1259405952 weight_movement_bytes=75497472 total_nvlink_bytes=1334903424 weight_to_token_ratio=0.059947 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.940021457e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=629506048 dispatch_meta_bytes=458864 combine_payload_bytes=629506048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629964912 ingress_bytes=705003520 link_bytes=705003520 token_bytes=1259470960 weight_movement_bytes=75497472 total_nvlink_bytes=1334968432 weight_to_token_ratio=0.059944 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.940437001e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87620 payload_rows_remote=76632 dup_rows=43452 dup_rows_remote=37948 dedup_ratio=0.331512 router_tokens_per_expert=[504, 491, 507, 528, 521, 503, 521, 477, 501, 514, 504, 494, 536, 564, 522, 506, 488, 494, 545, 493, 536, 532, 499, 512, 502, 505, 500, 514, 528, 506, 540, 486, 550, 491, 481, 513, 466, 516, 539, 494, 513, 521, 544, 506, 551, 545, 514, 502, 502, 472, 493, 510, 522, 511, 492, 486, 513, 519, 496, 509, 515, 513, 550, 485, 530, 502, 488, 469, 498, 537, 494, 520, 506, 538, 503, 557, 493, 524, 557, 503, 505, 527, 503, 511, 487, 495, 530, 512, 523, 489, 524, 516, 498, 510, 515, 510, 498, 511, 509, 512, 515, 487, 507, 518, 510, 501, 531, 553, 512, 537, 521, 498, 534, 487, 546, 512, 559, 476, 556, 534, 507, 515, 551, 492, 516, 500, 479, 471] payload_rows_per_dst=[10956, 11034, 10988, 10844, 10970, 10860, 11068, 10900] dup_rows_per_dst=[5430, 5326, 5504, 5332, 5468, 5450, 5372, 5570] combine_rows_per_src=[10956, 11034, 10988, 10844, 10970, 10860, 11068, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=458320 combine_payload_bytes=627769344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628227664 ingress_bytes=703266816 link_bytes=703266816 token_bytes=1255997008 weight_movement_bytes=75497472 total_nvlink_bytes=1331494480 weight_to_token_ratio=0.060110 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918413150e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87322 payload_rows_remote=76278 dup_rows=43750 dup_rows_remote=38126 dedup_ratio=0.333786 router_tokens_per_expert=[498, 560, 521, 509, 524, 521, 526, 535, 540, 505, 502, 556, 513, 508, 513, 495, 543, 502, 529, 506, 509, 506, 480, 502, 519, 483, 515, 534, 524, 499, 513, 514, 516, 507, 478, 513, 531, 488, 468, 520, 510, 511, 525, 533, 489, 540, 488, 493, 543, 503, 520, 468, 497, 533, 478, 529, 512, 495, 527, 535, 509, 493, 511, 520, 496, 502, 459, 502, 506, 487, 509, 490, 519, 527, 532, 537, 490, 476, 527, 546, 542, 493, 517, 535, 484, 525, 519, 502, 498, 484, 474, 501, 510, 533, 488, 534, 529, 490, 496, 529, 516, 533, 506, 530, 536, 530, 521, 474, 502, 529, 521, 510, 468, 540, 530, 528, 526, 525, 515, 538, 503, 511, 508, 487, 537, 510, 538, 489] payload_rows_per_dst=[11044, 11146, 11256, 10976, 10956, 10560, 10860, 10524] dup_rows_per_dst=[5624, 5976, 5984, 5360, 5254, 4952, 5638, 4962] combine_rows_per_src=[11044, 11146, 11256, 10976, 10956, 10560, 10860, 10524] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87362 payload_rows_remote=76386 dup_rows=43710 dup_rows_remote=38180 dedup_ratio=0.333481 router_tokens_per_expert=[500, 499, 532, 526, 480, 506, 538, 507, 471, 516, 511, 499, 473, 523, 496, 512, 516, 537, 549, 516, 508, 481, 524, 501, 508, 502, 501, 471, 518, 508, 519, 524, 531, 522, 514, 549, 534, 520, 500, 542, 493, 531, 466, 515, 501, 474, 490, 517, 494, 542, 528, 489, 517, 525, 482, 529, 547, 507, 523, 468, 514, 495, 554, 481, 531, 567, 537, 485, 501, 519, 535, 529, 513, 498, 476, 498, 502, 509, 501, 525, 499, 534, 526, 531, 486, 480, 511, 497, 497, 500, 521, 532, 526, 519, 516, 484, 520, 499, 550, 498, 499, 542, 533, 510, 487, 532, 538, 490, 507, 497, 509, 521, 554, 530, 546, 487, 489, 526, 491, 487, 493, 532, 518, 538, 548, 471, 491, 552] payload_rows_per_dst=[10870, 10846, 11008, 10932, 10860, 10892, 10978, 10976] dup_rows_per_dst=[5308, 5520, 5390, 5446, 5592, 5426, 5498, 5530] combine_rows_per_src=[10870, 10846, 11008, 10932, 10860, 10892, 10978, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87476 payload_rows_remote=76620 dup_rows=43596 dup_rows_remote=38218 dedup_ratio=0.332611 router_tokens_per_expert=[499, 505, 516, 514, 555, 527, 503, 516, 514, 472, 549, 501, 527, 514, 496, 508, 504, 550, 489, 553, 513, 488, 527, 501, 475, 496, 470, 538, 502, 495, 497, 519, 487, 489, 494, 530, 532, 504, 465, 524, 519, 534, 523, 537, 478, 504, 505, 507, 508, 494, 516, 498, 485, 510, 502, 533, 518, 503, 552, 519, 492, 548, 534, 499, 537, 510, 481, 506, 493, 525, 552, 500, 522, 477, 490, 509, 506, 536, 521, 529, 525, 531, 505, 525, 506, 519, 487, 534, 504, 579, 533, 502, 489, 528, 495, 530, 490, 481, 516, 520, 499, 514, 537, 516, 475, 521, 525, 532, 500, 512, 484, 502, 497, 508, 524, 522, 509, 498, 508, 529, 515, 522, 539, 529, 497, 533, 523, 497] payload_rows_per_dst=[10904, 10856, 10956, 10932, 10896, 11068, 10836, 11028] dup_rows_per_dst=[5528, 5378, 5308, 5490, 5492, 5516, 5412, 5472] combine_rows_per_src=[10904, 10856, 10956, 10932, 10896, 11068, 10836, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87126 payload_rows_remote=76180 dup_rows=43946 dup_rows_remote=38254 dedup_ratio=0.335281 router_tokens_per_expert=[502, 508, 533, 508, 523, 516, 530, 530, 538, 508, 472, 527, 504, 492, 527, 488, 512, 518, 497, 557, 491, 492, 516, 538, 516, 501, 517, 521, 463, 542, 519, 478, 525, 530, 506, 516, 489, 496, 519, 493, 476, 500, 539, 510, 509, 530, 496, 502, 491, 533, 516, 552, 521, 507, 498, 521, 500, 511, 535, 520, 521, 531, 507, 555, 501, 595, 529, 520, 507, 464, 504, 507, 528, 517, 513, 491, 495, 507, 503, 520, 528, 477, 506, 489, 501, 505, 548, 510, 507, 545, 510, 527, 506, 502, 516, 562, 528, 487, 452, 548, 485, 549, 509, 500, 519, 489, 521, 508, 454, 505, 497, 530, 525, 473, 493, 508, 518, 509, 538, 525, 506, 524, 501, 517, 530, 503, 523, 483] payload_rows_per_dst=[10894, 10898, 10868, 10946, 10920, 10926, 10818, 10856] dup_rows_per_dst=[5518, 5458, 5404, 5692, 5482, 5552, 5344, 5496] combine_rows_per_src=[10894, 10898, 10868, 10946, 10920, 10926, 10818, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87540 payload_rows_remote=76568 dup_rows=43532 dup_rows_remote=38150 dedup_ratio=0.332123 router_tokens_per_expert=[574, 509, 525, 535, 544, 550, 521, 455, 529, 512, 546, 512, 492, 553, 499, 520, 560, 490, 543, 538, 508, 486, 479, 523, 525, 525, 513, 499, 561, 548, 509, 489, 538, 500, 471, 513, 511, 534, 506, 503, 511, 519, 544, 512, 504, 512, 508, 499, 515, 510, 491, 506, 500, 535, 475, 553, 512, 498, 518, 539, 543, 498, 490, 539, 526, 513, 461, 486, 505, 489, 559, 490, 505, 523, 519, 539, 531, 551, 487, 521, 458, 513, 473, 463, 504, 541, 504, 502, 521, 479, 508, 524, 485, 532, 499, 471, 474, 512, 522, 518, 499, 544, 484, 536, 525, 519, 481, 496, 504, 524, 493, 546, 520, 485, 511, 496, 520, 514, 533, 514, 528, 500, 500, 512, 485, 499, 491, 490] payload_rows_per_dst=[11112, 11062, 10904, 10922, 10968, 10734, 10972, 10866] dup_rows_per_dst=[5640, 5530, 5466, 5522, 5442, 5220, 5382, 5330] combine_rows_per_src=[11112, 11062, 10904, 10922, 10968, 10734, 10972, 10866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87444 payload_rows_remote=76496 dup_rows=43628 dup_rows_remote=38078 dedup_ratio=0.332855 router_tokens_per_expert=[533, 503, 517, 502, 526, 499, 558, 528, 504, 542, 521, 475, 533, 501, 492, 551, 485, 478, 502, 507, 514, 512, 516, 527, 466, 499, 505, 488, 494, 522, 503, 501, 536, 483, 546, 484, 578, 504, 538, 512, 477, 476, 502, 519, 530, 522, 475, 506, 459, 499, 525, 497, 538, 558, 498, 524, 475, 517, 493, 499, 515, 503, 557, 502, 500, 529, 516, 539, 477, 504, 492, 519, 513, 527, 539, 557, 517, 499, 509, 473, 542, 522, 523, 455, 512, 532, 501, 516, 542, 492, 472, 495, 523, 543, 536, 543, 519, 522, 533, 508, 531, 566, 528, 507, 536, 486, 518, 507, 462, 499, 490, 479, 507, 517, 537, 515, 498, 520, 539, 514, 522, 562, 494, 479, 536, 476, 513, 506] payload_rows_per_dst=[10962, 10782, 10894, 10950, 11026, 10948, 10940, 10942] dup_rows_per_dst=[5608, 5256, 5482, 5368, 5394, 5550, 5442, 5528] combine_rows_per_src=[10962, 10782, 10894, 10950, 11026, 10948, 10940, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87368 payload_rows_remote=76524 dup_rows=43704 dup_rows_remote=38254 dedup_ratio=0.333435 router_tokens_per_expert=[526, 485, 533, 517, 480, 520, 500, 541, 491, 497, 514, 493, 529, 465, 533, 498, 516, 537, 510, 530, 477, 480, 567, 489, 491, 534, 484, 521, 513, 526, 561, 548, 514, 526, 533, 492, 516, 482, 489, 511, 516, 470, 489, 528, 491, 513, 503, 516, 532, 473, 506, 508, 504, 510, 519, 522, 543, 537, 458, 511, 495, 538, 492, 516, 555, 543, 531, 468, 476, 486, 527, 515, 499, 504, 517, 495, 480, 524, 500, 527, 489, 535, 515, 485, 541, 486, 542, 535, 502, 571, 498, 524, 501, 523, 515, 528, 518, 513, 531, 489, 498, 515, 491, 502, 511, 509, 531, 521, 527, 543, 499, 515, 543, 528, 514, 550, 497, 541, 488, 498, 485, 509, 513, 540, 511, 500, 463, 547] payload_rows_per_dst=[10838, 10992, 10812, 10884, 10844, 11012, 11008, 10978] dup_rows_per_dst=[5406, 5576, 5366, 5444, 5450, 5568, 5418, 5476] combine_rows_per_src=[10838, 10992, 10812, 10884, 10844, 11012, 11008, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=624869376 dispatch_meta_bytes=457616 combine_payload_bytes=624869376 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625326992 ingress_bytes=775864320 link_bytes=775864320 token_bytes=1250196368 weight_movement_bytes=150994944 total_nvlink_bytes=1401191312 weight_to_token_ratio=0.120777 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.839051689e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=458264 combine_payload_bytes=625754112 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=626212376 ingress_bytes=927744000 link_bytes=927744000 token_bytes=1251966488 weight_movement_bytes=301989888 total_nvlink_bytes=1553956376 weight_to_token_ratio=0.241212 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176509982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627671040 dispatch_meta_bytes=459352 combine_payload_bytes=627671040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628130392 ingress_bytes=703168512 link_bytes=703168512 token_bytes=1255801432 weight_movement_bytes=75497472 total_nvlink_bytes=1331298904 weight_to_token_ratio=0.060119 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917166517e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=624066560 dispatch_meta_bytes=457736 combine_payload_bytes=624066560 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624524296 ingress_bytes=775061504 link_bytes=775061504 token_bytes=1248590856 weight_movement_bytes=150994944 total_nvlink_bytes=1399585800 weight_to_token_ratio=0.120932 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.828870852e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627245056 dispatch_meta_bytes=458872 combine_payload_bytes=627245056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627703928 ingress_bytes=702742528 link_bytes=702742528 token_bytes=1254948984 weight_movement_bytes=75497472 total_nvlink_bytes=1330446456 weight_to_token_ratio=0.060160 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911764440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87468 payload_rows_remote=76600 dup_rows=43604 dup_rows_remote=38010 dedup_ratio=0.332672 router_tokens_per_expert=[507, 532, 485, 504, 512, 500, 521, 503, 525, 511, 522, 500, 476, 504, 507, 522, 502, 507, 505, 487, 532, 503, 534, 531, 506, 507, 476, 514, 513, 505, 504, 527, 511, 523, 484, 552, 498, 520, 497, 524, 510, 508, 509, 568, 501, 497, 502, 527, 530, 499, 534, 452, 511, 534, 487, 524, 482, 492, 513, 486, 484, 499, 510, 533, 528, 531, 492, 525, 532, 517, 539, 572, 496, 529, 531, 509, 478, 481, 499, 512, 504, 517, 515, 516, 500, 522, 526, 523, 460, 457, 532, 540, 493, 503, 520, 519, 493, 535, 524, 534, 525, 493, 530, 551, 520, 499, 543, 535, 531, 539, 521, 491, 502, 510, 541, 537, 471, 539, 491, 510, 536, 489, 476, 489, 524, 513, 520, 521] payload_rows_per_dst=[10874, 11000, 10868, 10846, 10984, 10952, 11034, 10910] dup_rows_per_dst=[5388, 5306, 5594, 5294, 5558, 5342, 5694, 5428] combine_rows_per_src=[10874, 11000, 10868, 10846, 10984, 10952, 11034, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626655232 dispatch_meta_bytes=458296 combine_payload_bytes=626655232 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627113528 ingress_bytes=777650176 link_bytes=777650176 token_bytes=1253768760 weight_movement_bytes=150994944 total_nvlink_bytes=1404763704 weight_to_token_ratio=0.120433 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861698857e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=459112 combine_payload_bytes=626884608 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627343720 ingress_bytes=702382080 link_bytes=702382080 token_bytes=1254228328 weight_movement_bytes=75497472 total_nvlink_bytes=1329725800 weight_to_token_ratio=0.060194 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907193452e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627507200 dispatch_meta_bytes=458440 combine_payload_bytes=627507200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627965640 ingress_bytes=703004672 link_bytes=703004672 token_bytes=1255472840 weight_movement_bytes=75497472 total_nvlink_bytes=1330970312 weight_to_token_ratio=0.060135 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915088795e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87340 payload_rows_remote=76516 dup_rows=43732 dup_rows_remote=38342 dedup_ratio=0.333649 router_tokens_per_expert=[481, 515, 478, 513, 468, 530, 511, 484, 499, 519, 518, 493, 539, 492, 524, 470, 562, 520, 506, 476, 487, 454, 483, 500, 509, 507, 510, 548, 517, 526, 524, 531, 543, 525, 516, 541, 507, 518, 548, 528, 511, 476, 509, 518, 504, 556, 514, 554, 499, 508, 511, 492, 516, 498, 517, 534, 523, 493, 502, 497, 520, 528, 490, 566, 504, 497, 496, 526, 511, 543, 472, 486, 507, 475, 527, 535, 534, 498, 497, 506, 489, 510, 511, 523, 548, 510, 481, 487, 506, 509, 537, 496, 554, 485, 536, 510, 528, 517, 522, 503, 526, 495, 497, 498, 548, 456, 547, 481, 543, 537, 519, 536, 509, 498, 545, 520, 489, 519, 522, 500, 491, 492, 506, 523, 572, 500, 529, 506] payload_rows_per_dst=[10824, 10960, 11082, 10878, 10958, 10828, 10978, 10832] dup_rows_per_dst=[5390, 5386, 5804, 5514, 5466, 5244, 5528, 5400] combine_rows_per_src=[10824, 10960, 11082, 10878, 10958, 10828, 10978, 10832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87330 payload_rows_remote=76346 dup_rows=43742 dup_rows_remote=38052 dedup_ratio=0.333725 router_tokens_per_expert=[511, 508, 509, 507, 499, 512, 490, 490, 527, 536, 456, 501, 480, 523, 529, 498, 506, 481, 535, 487, 531, 456, 507, 490, 539, 536, 485, 513, 481, 529, 518, 558, 487, 496, 551, 496, 526, 508, 526, 547, 511, 530, 506, 501, 492, 507, 519, 493, 512, 524, 485, 521, 528, 546, 523, 529, 540, 529, 495, 526, 516, 479, 523, 499, 474, 529, 495, 489, 513, 543, 559, 531, 489, 519, 510, 506, 485, 522, 533, 495, 501, 519, 535, 506, 512, 482, 512, 527, 491, 493, 532, 489, 505, 466, 547, 533, 527, 515, 530, 492, 540, 482, 504, 545, 512, 498, 512, 531, 515, 484, 510, 518, 511, 478, 500, 526, 519, 510, 514, 548, 501, 519, 518, 528, 521, 569, 499, 519] payload_rows_per_dst=[10942, 10880, 10872, 10952, 10810, 10902, 10988, 10984] dup_rows_per_dst=[5210, 5424, 5520, 5440, 5574, 5398, 5486, 5690] combine_rows_per_src=[10942, 10880, 10872, 10952, 10810, 10902, 10988, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87614 payload_rows_remote=76836 dup_rows=43458 dup_rows_remote=38000 dedup_ratio=0.331558 router_tokens_per_expert=[504, 511, 524, 500, 542, 481, 504, 551, 500, 510, 513, 526, 484, 492, 513, 502, 503, 512, 460, 511, 530, 502, 463, 517, 484, 526, 518, 534, 530, 500, 499, 529, 483, 473, 532, 501, 539, 544, 490, 507, 533, 553, 483, 527, 545, 505, 550, 497, 489, 522, 518, 531, 559, 465, 517, 572, 463, 486, 508, 498, 491, 484, 502, 523, 510, 525, 497, 525, 532, 498, 506, 516, 517, 521, 506, 508, 504, 517, 507, 511, 515, 503, 494, 533, 516, 518, 504, 512, 549, 545, 523, 498, 541, 514, 516, 514, 479, 534, 509, 506, 502, 495, 487, 461, 493, 502, 517, 495, 542, 535, 535, 513, 518, 507, 541, 517, 497, 523, 518, 504, 533, 476, 478, 489, 520, 522, 544, 584] payload_rows_per_dst=[10886, 10778, 11070, 10958, 10940, 11082, 10828, 11072] dup_rows_per_dst=[5428, 5458, 5454, 5298, 5460, 5508, 5382, 5470] combine_rows_per_src=[10886, 10778, 11070, 10958, 10940, 11082, 10828, 11072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87686 payload_rows_remote=76960 dup_rows=43386 dup_rows_remote=38112 dedup_ratio=0.331009 router_tokens_per_expert=[495, 565, 535, 514, 481, 504, 526, 552, 526, 562, 504, 547, 544, 516, 497, 496, 532, 512, 497, 499, 491, 543, 539, 505, 483, 539, 548, 519, 511, 520, 516, 495, 531, 513, 468, 529, 507, 487, 538, 481, 472, 518, 535, 523, 480, 518, 510, 501, 551, 514, 471, 500, 503, 551, 528, 506, 507, 494, 490, 498, 533, 523, 517, 506, 495, 530, 552, 522, 492, 500, 534, 481, 499, 555, 476, 518, 466, 520, 529, 562, 450, 550, 520, 533, 479, 517, 501, 499, 511, 493, 467, 494, 494, 510, 477, 502, 493, 464, 487, 524, 498, 520, 507, 485, 488, 515, 534, 514, 522, 479, 477, 493, 482, 558, 495, 524, 523, 534, 541, 511, 533, 537, 551, 496, 542, 520, 537, 508] payload_rows_per_dst=[11070, 11018, 10866, 11064, 11058, 10774, 10726, 11110] dup_rows_per_dst=[5658, 5480, 5356, 5320, 5404, 5220, 5274, 5674] combine_rows_per_src=[11070, 11018, 10866, 11064, 11058, 10774, 10726, 11110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87456 payload_rows_remote=76590 dup_rows=43616 dup_rows_remote=37992 dedup_ratio=0.332764 router_tokens_per_expert=[495, 485, 495, 489, 497, 517, 512, 491, 516, 513, 534, 518, 518, 509, 513, 529, 495, 540, 516, 519, 488, 507, 524, 510, 501, 502, 496, 521, 492, 506, 548, 531, 475, 474, 555, 502, 505, 485, 522, 530, 486, 484, 492, 460, 519, 502, 492, 523, 489, 498, 512, 497, 528, 538, 513, 535, 554, 518, 516, 496, 507, 501, 552, 491, 510, 524, 501, 540, 516, 523, 489, 489, 517, 508, 499, 514, 539, 472, 515, 515, 488, 520, 532, 516, 502, 518, 501, 524, 521, 481, 514, 531, 505, 553, 539, 508, 543, 490, 509, 519, 521, 535, 513, 523, 501, 485, 572, 520, 510, 523, 542, 522, 489, 510, 517, 516, 492, 513, 516, 508, 483, 535, 503, 536, 528, 508, 500, 552] payload_rows_per_dst=[10966, 10992, 10726, 10866, 10910, 11058, 11012, 10926] dup_rows_per_dst=[5296, 5400, 5286, 5624, 5432, 5448, 5644, 5486] combine_rows_per_src=[10966, 10992, 10726, 10866, 10910, 11058, 11012, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87106 payload_rows_remote=76170 dup_rows=43966 dup_rows_remote=38516 dedup_ratio=0.335434 router_tokens_per_expert=[483, 545, 497, 503, 513, 553, 507, 498, 486, 515, 502, 485, 512, 501, 540, 520, 467, 502, 512, 475, 543, 516, 481, 502, 522, 530, 523, 469, 542, 514, 535, 527, 516, 518, 474, 486, 529, 513, 491, 503, 509, 504, 536, 519, 535, 524, 532, 510, 513, 478, 501, 520, 547, 514, 511, 514, 518, 530, 534, 537, 549, 518, 549, 492, 503, 519, 529, 492, 486, 482, 529, 532, 521, 550, 502, 522, 494, 514, 502, 516, 499, 524, 528, 493, 462, 544, 530, 495, 490, 522, 502, 541, 491, 470, 482, 493, 504, 507, 536, 477, 509, 535, 508, 501, 520, 505, 512, 553, 511, 525, 515, 518, 531, 546, 526, 493, 478, 496, 513, 518, 503, 504, 503, 490, 577, 489, 524, 506] payload_rows_per_dst=[10800, 10814, 10884, 11052, 10936, 10808, 10914, 10898] dup_rows_per_dst=[5520, 5506, 5514, 5598, 5450, 5324, 5558, 5496] combine_rows_per_src=[10800, 10814, 10884, 11052, 10936, 10808, 10914, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76520 dup_rows=43492 dup_rows_remote=37776 dedup_ratio=0.331818 router_tokens_per_expert=[548, 516, 533, 565, 461, 509, 545, 507, 517, 509, 523, 511, 479, 485, 494, 525, 544, 561, 513, 490, 512, 494, 491, 496, 508, 529, 499, 527, 484, 507, 489, 481, 513, 483, 526, 544, 518, 473, 497, 509, 517, 541, 503, 526, 512, 490, 492, 462, 506, 513, 536, 481, 525, 484, 512, 494, 494, 529, 524, 503, 500, 522, 517, 507, 502, 515, 498, 491, 514, 527, 504, 511, 478, 517, 539, 511, 515, 499, 486, 525, 508, 529, 520, 518, 573, 522, 536, 546, 506, 537, 545, 504, 542, 497, 487, 518, 529, 507, 493, 506, 469, 539, 526, 491, 520, 526, 505, 537, 499, 520, 527, 536, 505, 484, 509, 531, 476, 504, 513, 508, 485, 531, 558, 504, 518, 511, 513, 531] payload_rows_per_dst=[10974, 10842, 10860, 10916, 10994, 11060, 10940, 10994] dup_rows_per_dst=[5480, 5408, 5352, 5378, 5270, 5716, 5520, 5368] combine_rows_per_src=[10974, 10842, 10860, 10916, 10994, 11060, 10940, 10994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626819072 dispatch_meta_bytes=459432 combine_payload_bytes=626819072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627278504 ingress_bytes=702316544 link_bytes=702316544 token_bytes=1254097576 weight_movement_bytes=75497472 total_nvlink_bytes=1329595048 weight_to_token_ratio=0.060201 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906362363e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625426432 dispatch_meta_bytes=457592 combine_payload_bytes=625426432 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625884024 ingress_bytes=851918848 link_bytes=851918848 token_bytes=1251310456 weight_movement_bytes=226492416 total_nvlink_bytes=1477802872 weight_to_token_ratio=0.181004 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080353016e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=629440512 dispatch_meta_bytes=459344 combine_payload_bytes=629440512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629899856 ingress_bytes=704937984 link_bytes=704937984 token_bytes=1259340368 weight_movement_bytes=75497472 total_nvlink_bytes=1334837840 weight_to_token_ratio=0.059950 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.939605912e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87212 payload_rows_remote=76386 dup_rows=43860 dup_rows_remote=38552 dedup_ratio=0.334625 router_tokens_per_expert=[546, 493, 566, 524, 519, 517, 507, 510, 521, 514, 480, 508, 552, 532, 467, 548, 518, 512, 503, 521, 537, 491, 484, 477, 540, 527, 503, 489, 525, 542, 506, 504, 503, 503, 483, 494, 528, 500, 492, 484, 490, 477, 507, 537, 542, 540, 455, 532, 503, 543, 514, 494, 533, 498, 504, 538, 516, 509, 523, 503, 453, 516, 520, 522, 534, 521, 512, 503, 488, 543, 507, 500, 478, 536, 495, 521, 513, 509, 554, 496, 487, 525, 508, 474, 508, 520, 505, 568, 506, 487, 549, 530, 533, 497, 492, 499, 509, 471, 523, 507, 522, 512, 521, 510, 533, 534, 504, 530, 541, 530, 555, 525, 507, 504, 492, 533, 513, 533, 490, 517, 498, 517, 510, 448, 489, 488, 523, 510] payload_rows_per_dst=[11026, 10882, 10826, 10872, 10876, 10908, 10972, 10850] dup_rows_per_dst=[5582, 5476, 5308, 5506, 5544, 5468, 5682, 5294] combine_rows_per_src=[11026, 10882, 10826, 10872, 10876, 10908, 10972, 10850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=630456320 dispatch_meta_bytes=460288 combine_payload_bytes=630456320 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=630916608 ingress_bytes=781451264 link_bytes=781451264 token_bytes=1261372928 weight_movement_bytes=150994944 total_nvlink_bytes=1412367872 weight_to_token_ratio=0.119707 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.909902003e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627425280 dispatch_meta_bytes=458328 combine_payload_bytes=627425280 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627883608 ingress_bytes=853917696 link_bytes=853917696 token_bytes=1255308888 weight_movement_bytes=226492416 total_nvlink_bytes=1481801304 weight_to_token_ratio=0.180428 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082887837e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=623984640 dispatch_meta_bytes=458744 combine_payload_bytes=623984640 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624443384 ingress_bytes=774979584 link_bytes=774979584 token_bytes=1248428024 weight_movement_bytes=150994944 total_nvlink_bytes=1399422968 weight_to_token_ratio=0.120948 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.827831991e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=457184 combine_payload_bytes=626851840 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627309024 ingress_bytes=853344256 link_bytes=853344256 token_bytes=1254160864 weight_movement_bytes=226492416 total_nvlink_bytes=1480653280 weight_to_token_ratio=0.180593 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082160634e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=459752 combine_payload_bytes=625754112 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=626213864 ingress_bytes=927744000 link_bytes=927744000 token_bytes=1251967976 weight_movement_bytes=301989888 total_nvlink_bytes=1553957864 weight_to_token_ratio=0.241212 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176509982e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76362 dup_rows=43582 dup_rows_remote=37820 dedup_ratio=0.332504 router_tokens_per_expert=[512, 549, 520, 519, 484, 472, 509, 503, 529, 484, 524, 505, 503, 530, 490, 543, 525, 536, 499, 535, 482, 542, 519, 526, 527, 496, 483, 495, 489, 537, 510, 505, 464, 525, 553, 527, 476, 513, 529, 527, 489, 515, 500, 537, 489, 552, 484, 491, 557, 540, 509, 516, 506, 466, 482, 506, 511, 513, 523, 553, 519, 516, 476, 483, 490, 514, 538, 511, 535, 489, 484, 559, 527, 510, 535, 523, 499, 481, 490, 491, 490, 500, 532, 560, 476, 541, 494, 527, 540, 520, 494, 531, 498, 531, 499, 515, 482, 501, 504, 486, 532, 511, 519, 472, 475, 525, 554, 516, 504, 499, 517, 500, 501, 465, 500, 506, 522, 489, 504, 538, 519, 569, 534, 530, 531, 531, 523, 524] payload_rows_per_dst=[11128, 10920, 10976, 10926, 10924, 11250, 10736, 10630] dup_rows_per_dst=[5762, 5518, 5738, 5400, 5166, 5860, 5182, 4956] combine_rows_per_src=[11128, 10920, 10976, 10926, 10924, 11250, 10736, 10630] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87444 payload_rows_remote=76174 dup_rows=43628 dup_rows_remote=37586 dedup_ratio=0.332855 router_tokens_per_expert=[521, 550, 503, 503, 480, 494, 529, 551, 479, 513, 508, 459, 533, 490, 542, 567, 539, 506, 514, 519, 504, 510, 516, 508, 501, 488, 488, 480, 524, 530, 497, 495, 520, 519, 492, 522, 497, 515, 509, 530, 504, 502, 538, 557, 538, 514, 533, 505, 537, 528, 540, 529, 488, 496, 514, 503, 542, 521, 492, 542, 544, 519, 550, 567, 495, 513, 540, 504, 486, 466, 514, 506, 493, 510, 498, 503, 499, 494, 517, 503, 536, 487, 488, 505, 514, 534, 515, 540, 474, 514, 494, 437, 519, 504, 512, 520, 523, 489, 555, 516, 516, 530, 515, 538, 526, 518, 531, 506, 509, 479, 508, 466, 517, 536, 519, 478, 502, 482, 527, 501, 497, 502, 530, 497, 507, 518, 498, 518] payload_rows_per_dst=[10738, 10838, 11006, 10830, 11026, 10782, 10954, 11270] dup_rows_per_dst=[5132, 5400, 5280, 5200, 5630, 5408, 5536, 6042] combine_rows_per_src=[10738, 10838, 11006, 10830, 11026, 10782, 10954, 11270] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76540 dup_rows=43582 dup_rows_remote=38256 dedup_ratio=0.332504 router_tokens_per_expert=[500, 505, 531, 501, 495, 535, 466, 499, 505, 537, 521, 475, 510, 513, 509, 516, 524, 471, 488, 517, 516, 531, 537, 484, 535, 552, 504, 510, 489, 480, 506, 494, 540, 484, 535, 498, 540, 514, 515, 533, 517, 489, 502, 517, 534, 530, 445, 482, 448, 493, 507, 507, 506, 527, 516, 499, 497, 506, 474, 512, 525, 533, 496, 500, 540, 535, 508, 557, 498, 499, 491, 505, 513, 485, 503, 512, 523, 515, 498, 510, 548, 531, 575, 475, 501, 508, 548, 518, 537, 534, 525, 549, 483, 481, 521, 519, 536, 508, 540, 551, 515, 507, 514, 546, 493, 495, 528, 488, 537, 507, 494, 500, 520, 524, 495, 515, 512, 488, 525, 520, 503, 509, 532, 553, 478, 531, 524, 526] payload_rows_per_dst=[10790, 10950, 10980, 10792, 10900, 11076, 10920, 11082] dup_rows_per_dst=[5446, 5326, 5370, 5300, 5484, 5630, 5598, 5428] combine_rows_per_src=[10790, 10950, 10980, 10792, 10900, 11076, 10920, 11082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15066626096 prefetch_bytes=679477248 reduce_bytes=981467136 weight_movement_bytes=1660944384 egress_bytes=7536064560 ingress_bytes=9191505920 link_bytes=9191505920 total_nvlink_bytes=16727570480 weight_to_token_ratio=0.110240 prefetch_share_of_ingress=0.073924 dedup_ratio=0.332236 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=13 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044271 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.713436091e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87542 payload_rows_remote=76568 dup_rows=43530 dup_rows_remote=38026 dedup_ratio=0.332108 router_tokens_per_expert=[510, 495, 528, 490, 493, 497, 515, 510, 517, 518, 535, 475, 548, 539, 564, 528, 508, 497, 516, 533, 511, 485, 514, 512, 507, 522, 514, 517, 529, 501, 503, 509, 552, 480, 490, 507, 508, 522, 516, 483, 510, 540, 520, 520, 483, 537, 509, 518, 505, 531, 554, 503, 502, 474, 538, 532, 538, 559, 503, 499, 473, 505, 517, 480, 521, 515, 516, 508, 583, 529, 495, 482, 477, 501, 518, 507, 513, 475, 495, 517, 513, 501, 513, 465, 485, 549, 479, 517, 498, 502, 511, 567, 546, 530, 515, 486, 539, 486, 510, 522, 529, 539, 494, 499, 482, 501, 487, 554, 508, 517, 545, 527, 504, 511, 486, 520, 482, 546, 492, 519, 467, 513, 508, 511, 506, 520, 495, 540] payload_rows_per_dst=[10946, 10954, 10968, 10966, 10932, 10910, 10974, 10892] dup_rows_per_dst=[5578, 5402, 5422, 5460, 5372, 5444, 5504, 5348] combine_rows_per_src=[10946, 10954, 10968, 10966, 10932, 10910, 10974, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76620 dup_rows=43558 dup_rows_remote=38126 dedup_ratio=0.332321 router_tokens_per_expert=[492, 458, 479, 480, 512, 519, 483, 509, 520, 562, 525, 502, 519, 517, 525, 526, 501, 516, 543, 536, 510, 500, 484, 491, 540, 518, 512, 493, 521, 485, 514, 535, 512, 493, 525, 530, 511, 550, 531, 520, 522, 494, 554, 493, 559, 519, 529, 476, 462, 511, 533, 502, 506, 496, 514, 514, 478, 538, 525, 528, 494, 482, 534, 546, 524, 529, 520, 518, 492, 491, 548, 501, 518, 498, 526, 484, 515, 498, 478, 556, 522, 500, 498, 508, 515, 538, 496, 528, 521, 563, 477, 528, 521, 533, 505, 533, 510, 487, 513, 474, 502, 522, 503, 535, 487, 497, 494, 512, 501, 553, 502, 523, 564, 535, 508, 464, 510, 502, 515, 553, 524, 508, 514, 470, 475, 503, 529, 457] payload_rows_per_dst=[10842, 10990, 11006, 10894, 11000, 10976, 10942, 10864] dup_rows_per_dst=[5414, 5408, 5630, 5432, 5392, 5596, 5288, 5398] combine_rows_per_src=[10842, 10990, 11006, 10894, 11000, 10976, 10942, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87494 payload_rows_remote=76618 dup_rows=43578 dup_rows_remote=38092 dedup_ratio=0.332474 router_tokens_per_expert=[507, 490, 501, 530, 515, 495, 521, 555, 517, 510, 523, 524, 561, 528, 529, 510, 544, 529, 508, 492, 502, 523, 510, 483, 508, 531, 493, 500, 540, 486, 524, 520, 503, 553, 521, 513, 487, 505, 484, 523, 478, 508, 518, 498, 499, 522, 508, 520, 502, 525, 563, 506, 492, 518, 557, 522, 488, 494, 476, 541, 497, 496, 502, 497, 506, 496, 538, 520, 517, 472, 476, 478, 537, 519, 524, 500, 508, 537, 540, 513, 531, 503, 510, 506, 508, 524, 553, 545, 512, 484, 457, 525, 525, 533, 526, 519, 524, 492, 546, 534, 506, 495, 548, 527, 501, 535, 505, 516, 505, 503, 498, 502, 471, 527, 532, 519, 526, 515, 467, 499, 492, 521, 501, 497, 475, 463, 545, 482] payload_rows_per_dst=[11028, 10976, 10846, 11054, 10876, 11032, 10932, 10750] dup_rows_per_dst=[5604, 5410, 5434, 5298, 5486, 5490, 5542, 5314] combine_rows_per_src=[11028, 10976, 10846, 11054, 10876, 11032, 10932, 10750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76748 dup_rows=43514 dup_rows_remote=38178 dedup_ratio=0.331985 router_tokens_per_expert=[514, 508, 509, 520, 517, 469, 474, 524, 490, 507, 531, 496, 470, 522, 490, 531, 545, 534, 518, 485, 514, 540, 549, 527, 495, 484, 465, 490, 508, 543, 537, 555, 536, 461, 484, 510, 503, 517, 518, 520, 479, 492, 510, 567, 523, 497, 524, 533, 520, 492, 487, 550, 500, 525, 506, 500, 543, 524, 478, 528, 538, 498, 503, 525, 513, 513, 487, 504, 535, 509, 544, 507, 493, 546, 515, 531, 512, 528, 505, 491, 488, 497, 534, 490, 530, 512, 495, 513, 476, 525, 520, 496, 504, 509, 509, 475, 506, 529, 508, 491, 506, 514, 527, 557, 499, 501, 541, 534, 534, 541, 488, 531, 503, 534, 442, 515, 539, 513, 516, 497, 503, 524, 537, 452, 515, 528, 544, 509] payload_rows_per_dst=[10868, 10988, 10972, 10978, 11016, 10810, 10994, 10932] dup_rows_per_dst=[5276, 5590, 5376, 5456, 5450, 5336, 5620, 5410] combine_rows_per_src=[10868, 10988, 10972, 10978, 11016, 10810, 10994, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625557504 dispatch_meta_bytes=456728 combine_payload_bytes=625557504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626014232 ingress_bytes=701054976 link_bytes=701054976 token_bytes=1251571736 weight_movement_bytes=75497472 total_nvlink_bytes=1327069208 weight_to_token_ratio=0.060322 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890363905e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=459184 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627474864 ingress_bytes=702513152 link_bytes=702513152 token_bytes=1254490544 weight_movement_bytes=75497472 total_nvlink_bytes=1329988016 weight_to_token_ratio=0.060182 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908855629e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=624017408 dispatch_meta_bytes=455040 combine_payload_bytes=624017408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624472448 ingress_bytes=699514880 link_bytes=699514880 token_bytes=1248489856 weight_movement_bytes=75497472 total_nvlink_bytes=1323987328 weight_to_token_ratio=0.060471 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.870833320e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627245056 dispatch_meta_bytes=458376 combine_payload_bytes=627245056 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627703432 ingress_bytes=778240000 link_bytes=778240000 token_bytes=1254948488 weight_movement_bytes=150994944 total_nvlink_bytes=1405943432 weight_to_token_ratio=0.120320 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.869178656e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627671040 dispatch_meta_bytes=458984 combine_payload_bytes=627671040 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628130024 ingress_bytes=778665984 link_bytes=778665984 token_bytes=1255801064 weight_movement_bytes=150994944 total_nvlink_bytes=1406796008 weight_to_token_ratio=0.120238 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874580732e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627654656 dispatch_meta_bytes=458840 combine_payload_bytes=627654656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628113496 ingress_bytes=703152128 link_bytes=703152128 token_bytes=1255768152 weight_movement_bytes=75497472 total_nvlink_bytes=1331265624 weight_to_token_ratio=0.060121 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916958744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628719616 dispatch_meta_bytes=459704 combine_payload_bytes=628719616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629179320 ingress_bytes=704217088 link_bytes=704217088 token_bytes=1257898936 weight_movement_bytes=75497472 total_nvlink_bytes=1333396408 weight_to_token_ratio=0.060019 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930463936e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15056503472 prefetch_bytes=830472192 reduce_bytes=528482304 weight_movement_bytes=1358954496 egress_bytes=7531004592 ingress_bytes=8884453376 link_bytes=8884453376 total_nvlink_bytes=16415457968 weight_to_token_ratio=0.090257 prefetch_share_of_ingress=0.093475 dedup_ratio=0.332553 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=7 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042126 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.388947885e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15040533672 prefetch_bytes=679477248 reduce_bytes=1132462080 weight_movement_bytes=1811939328 egress_bytes=7523013800 ingress_bytes=9329459200 link_bytes=9329459200 total_nvlink_bytes=16852473000 weight_to_token_ratio=0.120470 prefetch_share_of_ingress=0.072831 dedup_ratio=0.332497 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=15 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.859222905e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15069184688 prefetch_bytes=754974720 reduce_bytes=679477248 weight_movement_bytes=1434451968 egress_bytes=7537345200 ingress_bytes=8966291456 link_bytes=8966291456 total_nvlink_bytes=16503636656 weight_to_token_ratio=0.095191 prefetch_share_of_ingress=0.084201 dedup_ratio=0.332035 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=9 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044271 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.475433056e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15060138792 prefetch_bytes=905969664 reduce_bytes=301989888 weight_movement_bytes=1207959552 egress_bytes=7532821288 ingress_bytes=8735277056 link_bytes=8735277056 total_nvlink_bytes=16268098344 weight_to_token_ratio=0.080209 prefetch_share_of_ingress=0.103714 dedup_ratio=0.332458 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=4 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043045 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.231300742e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15051290928 prefetch_bytes=830472192 reduce_bytes=1132462080 weight_movement_bytes=1962934272 egress_bytes=7528397104 ingress_bytes=9485828096 link_bytes=9485828096 total_nvlink_bytes=17014225200 weight_to_token_ratio=0.130416 prefetch_share_of_ingress=0.087549 dedup_ratio=0.332575 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=15 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.002447105e-04 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76468 dup_rows=43584 dup_rows_remote=38004 dedup_ratio=0.332520 router_tokens_per_expert=[511, 532, 530, 505, 529, 483, 527, 537, 542, 522, 523, 505, 529, 517, 483, 503, 476, 527, 455, 469, 505, 510, 520, 506, 538, 478, 483, 488, 531, 521, 503, 487, 536, 496, 495, 506, 529, 499, 522, 496, 530, 543, 516, 504, 542, 493, 540, 528, 543, 553, 521, 508, 506, 499, 534, 500, 545, 533, 529, 529, 522, 527, 515, 518, 504, 498, 518, 505, 501, 512, 537, 522, 520, 503, 538, 547, 471, 511, 524, 488, 522, 490, 476, 498, 541, 507, 538, 523, 493, 512, 483, 510, 471, 519, 525, 481, 495, 527, 484, 539, 545, 496, 535, 524, 520, 514, 541, 505, 521, 490, 533, 488, 529, 479, 483, 496, 525, 496, 512, 517, 475, 532, 519, 520, 521, 459, 508, 488] payload_rows_per_dst=[11020, 11004, 10972, 11114, 10832, 10862, 10794, 10890] dup_rows_per_dst=[5580, 5554, 5534, 5650, 5566, 5316, 5156, 5228] combine_rows_per_src=[11020, 11004, 10972, 11114, 10832, 10862, 10794, 10890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=457888 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626883744 ingress_bytes=777420800 link_bytes=777420800 token_bytes=1253309600 weight_movement_bytes=150994944 total_nvlink_bytes=1404304544 weight_to_token_ratio=0.120477 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.858790047e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87590 payload_rows_remote=76804 dup_rows=43482 dup_rows_remote=38094 dedup_ratio=0.331741 router_tokens_per_expert=[522, 523, 534, 489, 560, 510, 475, 514, 500, 539, 523, 503, 516, 487, 476, 502, 510, 483, 512, 477, 519, 526, 547, 515, 534, 526, 501, 537, 532, 488, 524, 520, 501, 499, 497, 523, 518, 525, 542, 466, 519, 509, 523, 525, 497, 525, 564, 512, 532, 518, 530, 541, 557, 513, 492, 515, 496, 510, 521, 536, 530, 550, 476, 535, 493, 481, 514, 497, 539, 534, 475, 521, 501, 507, 500, 504, 488, 509, 509, 506, 505, 521, 515, 561, 526, 517, 503, 479, 461, 509, 521, 525, 488, 484, 514, 458, 503, 521, 462, 576, 521, 494, 540, 478, 507, 519, 505, 501, 479, 523, 509, 501, 548, 529, 512, 512, 479, 462, 530, 500, 509, 524, 516, 537, 544, 489, 522, 498] payload_rows_per_dst=[10912, 11004, 10988, 11140, 10900, 10786, 10922, 10938] dup_rows_per_dst=[5434, 5498, 5502, 5564, 5256, 5388, 5356, 5484] combine_rows_per_src=[10912, 11004, 10988, 11140, 10900, 10786, 10922, 10938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15025621680 prefetch_bytes=754974720 reduce_bytes=754974720 weight_movement_bytes=1509949440 egress_bytes=7515556528 ingress_bytes=9020014592 link_bytes=9020014592 total_nvlink_bytes=16535571120 weight_to_token_ratio=0.100492 prefetch_share_of_ingress=0.083700 dedup_ratio=0.332961 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=10 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.532206805e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=6 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=629178368 dispatch_meta_bytes=459592 combine_payload_bytes=629178368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629637960 ingress_bytes=704675840 link_bytes=704675840 token_bytes=1258816328 weight_movement_bytes=75497472 total_nvlink_bytes=1334313800 weight_to_token_ratio=0.059975 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.936281557e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=6 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15057091376 prefetch_bytes=905969664 reduce_bytes=830472192 weight_movement_bytes=1736441856 egress_bytes=7531297584 ingress_bytes=9262235648 link_bytes=9262235648 total_nvlink_bytes=16793533232 weight_to_token_ratio=0.115324 prefetch_share_of_ingress=0.097813 dedup_ratio=0.332389 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=11 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.044118 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.788182133e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none + [2026-08-29 23:36:24] iteration 7/ 12 | consumed samples: 56 | elapsed time per iteration (ms): 357.4 | throughput per GPU (TFLOP/s/GPU): 505.9 | learning rate: 3.250000E-05 | global batch size: 8 | lm loss: 1.024783E+01 | loss scale: 1.0 | grad norm: 6.239 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87916 payload_rows_remote=76898 dup_rows=43156 dup_rows_remote=37608 dedup_ratio=0.329254 router_tokens_per_expert=[481, 512, 460, 475, 544, 512, 506, 489, 473, 543, 553, 471, 541, 490, 510, 506, 493, 543, 493, 469, 519, 495, 515, 538, 507, 521, 521, 506, 583, 514, 529, 537, 510, 490, 540, 529, 481, 507, 547, 531, 544, 537, 502, 552, 503, 497, 506, 508, 539, 534, 549, 546, 521, 497, 506, 530, 527, 520, 510, 542, 476, 557, 503, 528, 542, 523, 496, 503, 504, 508, 531, 503, 518, 505, 490, 536, 496, 545, 517, 532, 545, 485, 508, 508, 488, 482, 506, 534, 525, 488, 518, 516, 520, 526, 509, 521, 512, 470, 527, 540, 495, 542, 456, 489, 489, 501, 495, 494, 513, 526, 519, 520, 490, 469, 496, 517, 499, 470, 503, 491, 518, 506, 528, 513, 531, 507, 479, 485] payload_rows_per_dst=[10842, 11018, 11016, 11196, 11112, 10996, 10864, 10872] dup_rows_per_dst=[5290, 5548, 5552, 5574, 5386, 5362, 5312, 5132] combine_rows_per_src=[10842, 11018, 11016, 11196, 11112, 10996, 10864, 10872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87468 payload_rows_remote=76560 dup_rows=43604 dup_rows_remote=38252 dedup_ratio=0.332672 router_tokens_per_expert=[525, 513, 486, 548, 531, 552, 508, 534, 523, 505, 507, 513, 544, 492, 521, 494, 508, 490, 554, 517, 498, 482, 514, 505, 534, 501, 515, 527, 526, 479, 519, 488, 536, 473, 523, 502, 498, 487, 510, 517, 533, 534, 545, 540, 513, 549, 474, 502, 517, 504, 511, 500, 490, 528, 513, 519, 507, 513, 495, 536, 535, 469, 509, 484, 501, 530, 502, 514, 498, 490, 489, 510, 521, 474, 523, 518, 490, 529, 519, 523, 478, 515, 485, 526, 525, 530, 501, 518, 524, 514, 463, 509, 520, 524, 479, 520, 519, 483, 476, 521, 584, 541, 504, 548, 492, 510, 506, 536, 477, 539, 514, 490, 505, 483, 492, 514, 516, 512, 506, 510, 512, 505, 499, 549, 515, 547, 530, 520] payload_rows_per_dst=[10998, 10804, 11042, 10908, 10884, 10900, 10920, 11012] dup_rows_per_dst=[5594, 5510, 5430, 5352, 5378, 5362, 5560, 5418] combine_rows_per_src=[10998, 10804, 11042, 10908, 10884, 10900, 10920, 11012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87468 payload_rows_remote=76640 dup_rows=43604 dup_rows_remote=38270 dedup_ratio=0.332672 router_tokens_per_expert=[531, 537, 537, 506, 488, 552, 521, 532, 517, 536, 506, 474, 508, 482, 494, 497, 537, 516, 523, 512, 511, 512, 540, 506, 476, 487, 537, 517, 548, 508, 545, 532, 480, 466, 500, 488, 498, 499, 522, 514, 512, 542, 526, 535, 522, 511, 513, 511, 512, 486, 525, 555, 538, 502, 511, 510, 517, 497, 499, 557, 467, 505, 514, 539, 515, 493, 469, 510, 527, 491, 505, 532, 526, 519, 545, 527, 538, 531, 529, 502, 497, 494, 511, 521, 518, 487, 465, 522, 458, 512, 499, 496, 545, 503, 487, 524, 507, 484, 503, 521, 490, 516, 538, 500, 511, 544, 526, 526, 482, 473, 463, 497, 514, 510, 513, 536, 546, 528, 477, 495, 539, 530, 496, 495, 486, 544, 523, 527] payload_rows_per_dst=[10936, 11026, 10890, 10994, 11028, 10862, 10828, 10904] dup_rows_per_dst=[5500, 5588, 5388, 5474, 5490, 5216, 5334, 5614] combine_rows_per_src=[10936, 11026, 10890, 10994, 11028, 10862, 10828, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87734 payload_rows_remote=76676 dup_rows=43338 dup_rows_remote=38026 dedup_ratio=0.330643 router_tokens_per_expert=[502, 551, 528, 525, 536, 487, 520, 502, 471, 525, 530, 514, 488, 551, 530, 522, 510, 500, 496, 534, 487, 508, 559, 523, 519, 477, 546, 536, 509, 558, 540, 529, 485, 486, 547, 506, 492, 520, 477, 516, 498, 510, 490, 516, 547, 508, 549, 538, 495, 508, 492, 520, 516, 489, 504, 456, 527, 494, 560, 504, 508, 556, 520, 525, 491, 529, 551, 512, 523, 498, 470, 491, 482, 540, 484, 471, 498, 502, 523, 528, 528, 508, 474, 529, 488, 501, 537, 520, 535, 495, 501, 494, 529, 477, 517, 504, 541, 495, 505, 493, 483, 510, 502, 478, 489, 538, 497, 542, 514, 519, 500, 500, 520, 524, 521, 518, 504, 510, 537, 533, 494, 536, 511, 531, 499, 510, 478, 502] payload_rows_per_dst=[11096, 10992, 11058, 10992, 10820, 10976, 10824, 10976] dup_rows_per_dst=[5468, 5670, 5312, 5356, 5366, 5298, 5388, 5480] combine_rows_per_src=[11096, 10992, 11058, 10992, 10820, 10976, 10824, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87308 payload_rows_remote=76204 dup_rows=43764 dup_rows_remote=37956 dedup_ratio=0.333893 router_tokens_per_expert=[547, 508, 526, 468, 505, 513, 515, 513, 485, 499, 523, 526, 484, 553, 489, 515, 517, 490, 496, 521, 513, 503, 513, 523, 504, 533, 524, 541, 484, 498, 496, 501, 504, 540, 525, 514, 489, 536, 526, 498, 481, 492, 488, 489, 490, 485, 527, 545, 493, 537, 546, 551, 483, 542, 516, 509, 567, 534, 515, 496, 510, 532, 518, 492, 497, 532, 518, 545, 500, 531, 482, 501, 531, 523, 507, 467, 520, 498, 496, 535, 469, 488, 504, 554, 523, 503, 502, 515, 523, 561, 517, 501, 497, 499, 527, 562, 490, 517, 554, 489, 518, 525, 483, 482, 519, 512, 514, 510, 505, 514, 494, 502, 544, 527, 514, 528, 507, 481, 479, 506, 504, 483, 504, 488, 533, 528, 551, 507] payload_rows_per_dst=[10870, 10428, 10950, 10896, 10890, 11118, 11052, 11104] dup_rows_per_dst=[5468, 4864, 5308, 5370, 5476, 5926, 5544, 5808] combine_rows_per_src=[10870, 10428, 10950, 10896, 10890, 11118, 11052, 11104] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=629948416 dispatch_meta_bytes=458024 combine_payload_bytes=629948416 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=630406440 ingress_bytes=931938304 link_bytes=931938304 token_bytes=1260354856 weight_movement_bytes=301989888 total_nvlink_bytes=1562344744 weight_to_token_ratio=0.239607 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.181828950e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627179520 dispatch_meta_bytes=459248 combine_payload_bytes=627179520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627638768 ingress_bytes=853671936 link_bytes=853671936 token_bytes=1254818288 weight_movement_bytes=226492416 total_nvlink_bytes=1481310704 weight_to_token_ratio=0.180498 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082576178e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627834880 dispatch_meta_bytes=459640 combine_payload_bytes=627834880 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628294520 ingress_bytes=703332352 link_bytes=703332352 token_bytes=1256129400 weight_movement_bytes=75497472 total_nvlink_bytes=1331626872 weight_to_token_ratio=0.060103 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919244238e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87390 payload_rows_remote=76450 dup_rows=43682 dup_rows_remote=38058 dedup_ratio=0.333267 router_tokens_per_expert=[500, 515, 529, 529, 510, 498, 490, 493, 502, 526, 466, 512, 483, 494, 501, 502, 521, 503, 528, 554, 538, 537, 496, 491, 526, 518, 520, 470, 502, 493, 493, 526, 501, 525, 549, 486, 531, 490, 515, 494, 524, 514, 543, 497, 524, 501, 537, 523, 532, 526, 536, 493, 534, 506, 532, 501, 495, 497, 538, 555, 504, 551, 525, 492, 523, 487, 564, 498, 533, 536, 520, 504, 501, 485, 563, 489, 521, 505, 523, 530, 553, 465, 500, 485, 512, 512, 510, 506, 518, 466, 522, 533, 526, 514, 499, 539, 512, 502, 517, 517, 511, 551, 485, 528, 468, 509, 477, 469, 526, 508, 505, 530, 507, 495, 482, 502, 511, 489, 514, 518, 538, 506, 482, 520, 516, 516, 509, 537] payload_rows_per_dst=[10824, 10876, 11036, 11056, 10940, 10920, 10816, 10922] dup_rows_per_dst=[5276, 5556, 5472, 5578, 5624, 5400, 5414, 5362] combine_rows_per_src=[10824, 10876, 11036, 11056, 10940, 10920, 10816, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87192 payload_rows_remote=76218 dup_rows=43880 dup_rows_remote=38412 dedup_ratio=0.334778 router_tokens_per_expert=[472, 523, 502, 514, 492, 497, 523, 493, 514, 510, 522, 502, 534, 467, 518, 503, 530, 575, 525, 530, 527, 501, 521, 506, 574, 531, 517, 489, 509, 473, 541, 523, 498, 500, 486, 489, 505, 523, 501, 492, 492, 504, 550, 544, 493, 523, 497, 500, 505, 511, 508, 511, 513, 526, 481, 543, 511, 510, 512, 520, 508, 465, 504, 500, 511, 535, 510, 498, 484, 484, 523, 558, 490, 509, 532, 508, 482, 553, 508, 518, 490, 500, 489, 540, 521, 537, 532, 538, 517, 500, 509, 551, 461, 513, 534, 489, 526, 547, 516, 501, 507, 560, 513, 524, 572, 505, 475, 518, 533, 497, 531, 494, 520, 509, 507, 482, 524, 540, 527, 486, 505, 504, 504, 487, 490, 533, 507, 485] payload_rows_per_dst=[10806, 11046, 10768, 10886, 10944, 10974, 10932, 10836] dup_rows_per_dst=[5366, 5698, 5426, 5370, 5462, 5468, 5706, 5384] combine_rows_per_src=[10806, 11046, 10768, 10886, 10944, 10974, 10932, 10836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=458808 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628588600 ingress_bytes=703627264 link_bytes=703627264 token_bytes=1256718392 weight_movement_bytes=75497472 total_nvlink_bytes=1332215864 weight_to_token_ratio=0.060075 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922984138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=624263168 dispatch_meta_bytes=456640 combine_payload_bytes=624263168 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624719808 ingress_bytes=699760640 link_bytes=699760640 token_bytes=1248982976 weight_movement_bytes=75497472 total_nvlink_bytes=1324480448 weight_to_token_ratio=0.060447 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.873949903e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87300 payload_rows_remote=76228 dup_rows=43772 dup_rows_remote=38106 dedup_ratio=0.333954 router_tokens_per_expert=[484, 542, 513, 483, 504, 509, 510, 549, 518, 537, 526, 510, 487, 536, 509, 493, 546, 489, 503, 507, 513, 521, 504, 537, 498, 514, 542, 505, 485, 517, 508, 528, 464, 497, 509, 499, 503, 514, 489, 532, 520, 551, 546, 511, 484, 544, 499, 506, 542, 567, 513, 485, 504, 522, 525, 525, 482, 496, 499, 480, 522, 534, 491, 507, 517, 461, 526, 516, 509, 493, 538, 520, 484, 513, 516, 553, 521, 506, 519, 495, 502, 537, 486, 492, 492, 565, 514, 497, 472, 519, 567, 511, 508, 494, 508, 483, 556, 520, 520, 490, 527, 517, 555, 495, 546, 533, 515, 479, 483, 544, 510, 499, 525, 492, 518, 563, 496, 522, 487, 497, 507, 538, 506, 468, 519, 519, 473, 494] payload_rows_per_dst=[11072, 10956, 10932, 10808, 10808, 10860, 10986, 10878] dup_rows_per_dst=[5666, 5492, 5492, 5262, 5464, 5434, 5592, 5370] combine_rows_per_src=[11072, 10956, 10932, 10808, 10808, 10860, 10986, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626278400 dispatch_meta_bytes=458032 combine_payload_bytes=626278400 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=626736432 ingress_bytes=777273344 link_bytes=777273344 token_bytes=1253014832 weight_movement_bytes=150994944 total_nvlink_bytes=1404009776 weight_to_token_ratio=0.120505 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.856920097e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=624377856 dispatch_meta_bytes=458520 combine_payload_bytes=624377856 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624836376 ingress_bytes=699875328 link_bytes=699875328 token_bytes=1249214232 weight_movement_bytes=75497472 total_nvlink_bytes=1324711704 weight_to_token_ratio=0.060436 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.875404308e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87554 payload_rows_remote=76704 dup_rows=43518 dup_rows_remote=38188 dedup_ratio=0.332016 router_tokens_per_expert=[495, 478, 470, 508, 502, 525, 495, 493, 538, 539, 523, 505, 547, 537, 509, 510, 533, 474, 491, 494, 495, 498, 486, 509, 496, 496, 475, 547, 499, 507, 542, 548, 494, 541, 511, 553, 494, 521, 532, 516, 491, 512, 496, 503, 498, 493, 522, 481, 470, 527, 512, 498, 536, 499, 536, 517, 522, 503, 502, 493, 555, 508, 521, 481, 545, 497, 516, 515, 497, 526, 512, 501, 509, 552, 507, 534, 564, 535, 452, 502, 482, 543, 507, 508, 490, 501, 509, 534, 508, 493, 514, 508, 521, 481, 512, 498, 556, 554, 557, 483, 538, 525, 554, 511, 522, 514, 527, 508, 485, 512, 537, 496, 524, 485, 525, 514, 511, 493, 508, 515, 548, 534, 525, 501, 483, 488, 494, 534] payload_rows_per_dst=[11028, 10850, 10882, 10986, 11002, 10900, 11050, 10856] dup_rows_per_dst=[5320, 5330, 5434, 5374, 5526, 5318, 5708, 5508] combine_rows_per_src=[11028, 10850, 10882, 10986, 11002, 10900, 11050, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87636 payload_rows_remote=76606 dup_rows=43436 dup_rows_remote=37838 dedup_ratio=0.331390 router_tokens_per_expert=[532, 506, 529, 511, 506, 558, 546, 493, 505, 512, 496, 527, 528, 512, 522, 518, 498, 520, 506, 486, 502, 507, 512, 503, 498, 473, 464, 490, 498, 537, 510, 566, 503, 526, 553, 532, 510, 499, 536, 506, 506, 517, 497, 505, 508, 540, 547, 504, 530, 542, 511, 567, 492, 508, 567, 512, 484, 550, 513, 527, 510, 491, 506, 504, 495, 520, 518, 494, 537, 482, 516, 539, 511, 515, 515, 505, 504, 528, 555, 526, 530, 488, 500, 491, 500, 464, 495, 519, 518, 491, 525, 505, 495, 484, 522, 528, 524, 514, 502, 501, 516, 546, 528, 516, 501, 520, 465, 516, 507, 503, 518, 481, 491, 486, 513, 498, 505, 520, 501, 483, 524, 553, 527, 483, 481, 544, 495, 485] payload_rows_per_dst=[11126, 10912, 10992, 11030, 10982, 10834, 10878, 10882] dup_rows_per_dst=[5476, 5228, 5586, 5598, 5538, 5276, 5438, 5296] combine_rows_per_src=[11126, 10912, 10992, 11030, 10982, 10834, 10878, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=624459776 dispatch_meta_bytes=457336 combine_payload_bytes=624459776 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624917112 ingress_bytes=699957248 link_bytes=699957248 token_bytes=1249376888 weight_movement_bytes=75497472 total_nvlink_bytes=1324874360 weight_to_token_ratio=0.060428 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.876443169e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76554 dup_rows=43570 dup_rows_remote=38074 dedup_ratio=0.332413 router_tokens_per_expert=[523, 531, 520, 542, 494, 505, 527, 481, 499, 538, 484, 468, 491, 507, 536, 498, 515, 498, 477, 493, 531, 521, 488, 526, 514, 531, 497, 531, 517, 524, 534, 475, 512, 489, 553, 498, 523, 548, 485, 489, 525, 500, 553, 496, 543, 499, 527, 510, 500, 498, 501, 547, 550, 488, 467, 533, 510, 500, 540, 533, 508, 505, 504, 536, 498, 521, 508, 517, 502, 482, 535, 522, 541, 481, 508, 523, 483, 539, 500, 515, 523, 514, 530, 495, 490, 506, 534, 518, 497, 535, 514, 520, 533, 495, 513, 542, 479, 539, 485, 501, 516, 515, 524, 500, 510, 490, 538, 512, 544, 519, 509, 541, 470, 486, 543, 535, 496, 501, 502, 567, 509, 465, 506, 511, 494, 525, 505, 479] payload_rows_per_dst=[10868, 10974, 10898, 11072, 10862, 11002, 10948, 10878] dup_rows_per_dst=[5420, 5370, 5602, 5368, 5488, 5516, 5496, 5310] combine_rows_per_src=[10868, 10974, 10898, 11072, 10862, 11002, 10948, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628359168 dispatch_meta_bytes=459568 combine_payload_bytes=628359168 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628818736 ingress_bytes=703856640 link_bytes=703856640 token_bytes=1257177904 weight_movement_bytes=75497472 total_nvlink_bytes=1332675376 weight_to_token_ratio=0.060053 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925892948e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87608 payload_rows_remote=76554 dup_rows=43464 dup_rows_remote=37926 dedup_ratio=0.331604 router_tokens_per_expert=[480, 490, 498, 546, 505, 500, 501, 531, 524, 506, 495, 527, 484, 503, 527, 551, 546, 512, 512, 511, 503, 500, 507, 524, 493, 536, 486, 520, 482, 526, 538, 509, 537, 497, 479, 524, 526, 517, 515, 487, 525, 546, 505, 532, 513, 497, 518, 538, 530, 478, 535, 535, 496, 519, 515, 498, 526, 526, 514, 499, 496, 546, 495, 521, 532, 511, 523, 477, 506, 526, 542, 506, 509, 523, 530, 505, 525, 527, 467, 529, 561, 484, 508, 500, 494, 533, 515, 510, 491, 530, 503, 518, 514, 492, 525, 506, 496, 525, 496, 540, 488, 514, 498, 483, 488, 488, 466, 535, 492, 564, 546, 483, 549, 516, 518, 516, 470, 526, 486, 527, 497, 483, 525, 500, 496, 513, 507, 525] payload_rows_per_dst=[10846, 11018, 11030, 11022, 10826, 11032, 10780, 11054] dup_rows_per_dst=[5476, 5392, 5482, 5448, 5218, 5762, 5148, 5538] combine_rows_per_src=[10846, 11018, 11030, 11022, 10826, 11032, 10780, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627556352 dispatch_meta_bytes=457776 combine_payload_bytes=627556352 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628014128 ingress_bytes=778551296 link_bytes=778551296 token_bytes=1255570480 weight_movement_bytes=150994944 total_nvlink_bytes=1406565424 weight_to_token_ratio=0.120260 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.873126327e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87630 payload_rows_remote=76592 dup_rows=43442 dup_rows_remote=37822 dedup_ratio=0.331436 router_tokens_per_expert=[490, 485, 496, 510, 520, 491, 495, 531, 540, 484, 549, 528, 491, 514, 533, 472, 498, 550, 504, 512, 531, 512, 534, 466, 527, 527, 501, 486, 506, 522, 495, 528, 527, 484, 523, 475, 541, 509, 518, 534, 529, 516, 540, 509, 554, 532, 504, 534, 553, 533, 511, 556, 504, 481, 523, 555, 529, 502, 491, 519, 488, 491, 525, 492, 532, 490, 507, 475, 496, 513, 516, 499, 525, 541, 511, 485, 521, 515, 495, 487, 497, 502, 513, 481, 556, 512, 531, 498, 524, 532, 594, 475, 526, 499, 457, 524, 498, 505, 501, 486, 477, 525, 529, 511, 513, 495, 509, 494, 524, 539, 518, 506, 547, 498, 516, 481, 534, 499, 517, 546, 515, 524, 524, 502, 500, 493, 488, 483] payload_rows_per_dst=[10922, 11032, 11038, 10946, 10808, 10974, 10932, 10978] dup_rows_per_dst=[5336, 5366, 5620, 5560, 5408, 5468, 5328, 5356] combine_rows_per_src=[10922, 11032, 11038, 10946, 10808, 10974, 10932, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627130368 dispatch_meta_bytes=458512 combine_payload_bytes=627130368 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627588880 ingress_bytes=778125312 link_bytes=778125312 token_bytes=1254719248 weight_movement_bytes=150994944 total_nvlink_bytes=1405714192 weight_to_token_ratio=0.120342 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.867724250e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627130368 dispatch_meta_bytes=457920 combine_payload_bytes=627130368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627588288 ingress_bytes=702627840 link_bytes=702627840 token_bytes=1254718656 weight_movement_bytes=75497472 total_nvlink_bytes=1330216128 weight_to_token_ratio=0.060171 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910310035e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87466 payload_rows_remote=76432 dup_rows=43606 dup_rows_remote=37992 dedup_ratio=0.332687 router_tokens_per_expert=[500, 507, 497, 526, 494, 527, 522, 510, 546, 472, 506, 524, 509, 507, 468, 543, 482, 532, 525, 462, 492, 528, 527, 493, 577, 534, 521, 521, 510, 451, 497, 515, 499, 492, 529, 547, 523, 480, 536, 515, 485, 494, 508, 482, 506, 503, 517, 559, 509, 512, 527, 539, 530, 481, 543, 531, 512, 495, 519, 456, 505, 523, 521, 464, 551, 558, 500, 492, 543, 509, 531, 497, 551, 502, 533, 532, 495, 486, 521, 523, 491, 499, 471, 508, 545, 538, 530, 493, 502, 509, 533, 518, 548, 523, 498, 497, 512, 536, 531, 497, 534, 500, 533, 510, 514, 519, 479, 520, 514, 475, 518, 488, 536, 521, 497, 515, 547, 501, 493, 503, 490, 515, 515, 483, 477, 543, 523, 503] payload_rows_per_dst=[10912, 10858, 10924, 10880, 11034, 10928, 10972, 10958] dup_rows_per_dst=[5404, 5476, 5426, 5454, 5614, 5478, 5388, 5366] combine_rows_per_src=[10912, 10858, 10924, 10880, 11034, 10928, 10972, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627441664 dispatch_meta_bytes=457656 combine_payload_bytes=627441664 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627899320 ingress_bytes=778436608 link_bytes=778436608 token_bytes=1255340984 weight_movement_bytes=150994944 total_nvlink_bytes=1406335928 weight_to_token_ratio=0.120282 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.871671922e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87420 payload_rows_remote=76584 dup_rows=43652 dup_rows_remote=38262 dedup_ratio=0.333038 router_tokens_per_expert=[507, 489, 506, 511, 491, 522, 526, 509, 519, 511, 459, 471, 537, 508, 496, 527, 527, 530, 525, 534, 507, 498, 483, 517, 508, 518, 492, 552, 497, 459, 494, 502, 504, 529, 496, 525, 521, 523, 521, 507, 505, 475, 473, 503, 535, 512, 544, 509, 506, 554, 520, 502, 480, 554, 495, 489, 543, 524, 529, 505, 500, 477, 530, 537, 506, 510, 547, 504, 526, 469, 530, 508, 493, 493, 519, 554, 497, 511, 528, 520, 519, 478, 531, 479, 518, 506, 496, 485, 538, 509, 516, 521, 547, 509, 478, 483, 540, 526, 516, 499, 538, 479, 506, 494, 534, 521, 537, 509, 516, 496, 551, 485, 497, 548, 519, 487, 508, 515, 515, 508, 519, 512, 527, 538, 555, 539, 488, 527] payload_rows_per_dst=[10824, 10900, 10968, 11040, 10922, 10836, 10900, 11030] dup_rows_per_dst=[5354, 5386, 5396, 5450, 5508, 5390, 5594, 5574] combine_rows_per_src=[10824, 10900, 10968, 11040, 10922, 10836, 10900, 11030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76236 dup_rows=43674 dup_rows_remote=38076 dedup_ratio=0.333206 router_tokens_per_expert=[517, 513, 535, 531, 520, 503, 505, 516, 506, 546, 517, 526, 534, 555, 527, 529, 519, 498, 494, 507, 536, 491, 488, 519, 478, 478, 512, 496, 502, 550, 484, 511, 522, 549, 509, 484, 482, 515, 538, 483, 508, 546, 523, 511, 498, 499, 509, 516, 485, 524, 506, 490, 529, 490, 475, 496, 534, 518, 517, 491, 500, 497, 499, 503, 520, 548, 491, 498, 510, 532, 500, 483, 523, 483, 490, 527, 505, 528, 515, 515, 495, 532, 516, 528, 481, 509, 502, 508, 516, 534, 504, 493, 520, 512, 505, 524, 522, 497, 486, 544, 532, 509, 490, 531, 539, 502, 546, 543, 519, 507, 481, 508, 488, 485, 512, 500, 494, 519, 545, 522, 543, 487, 524, 544, 522, 498, 542, 519] payload_rows_per_dst=[11162, 11134, 10746, 10722, 10828, 10896, 10922, 10988] dup_rows_per_dst=[5598, 5846, 5048, 5122, 5508, 5462, 5590, 5500] combine_rows_per_src=[11162, 11134, 10746, 10722, 10828, 10896, 10922, 10988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87462 payload_rows_remote=76578 dup_rows=43610 dup_rows_remote=38216 dedup_ratio=0.332718 router_tokens_per_expert=[553, 471, 484, 540, 503, 513, 526, 513, 559, 498, 494, 502, 541, 495, 494, 511, 461, 517, 468, 533, 516, 523, 547, 492, 538, 546, 453, 529, 502, 542, 471, 501, 541, 509, 468, 480, 503, 533, 538, 511, 544, 482, 510, 521, 521, 512, 470, 489, 554, 526, 521, 521, 504, 516, 533, 536, 532, 509, 505, 510, 514, 499, 516, 526, 472, 457, 505, 518, 517, 517, 491, 503, 492, 518, 568, 515, 482, 480, 514, 529, 512, 505, 519, 500, 496, 516, 546, 563, 465, 505, 519, 506, 500, 473, 514, 551, 507, 503, 536, 525, 509, 505, 528, 530, 520, 516, 502, 456, 543, 504, 474, 523, 513, 547, 507, 540, 534, 502, 520, 498, 510, 543, 535, 503, 515, 525, 492, 513] payload_rows_per_dst=[10958, 10884, 10944, 11052, 10876, 10886, 10840, 11022] dup_rows_per_dst=[5436, 5394, 5320, 5592, 5280, 5494, 5522, 5572] combine_rows_per_src=[10958, 10884, 10944, 11052, 10876, 10886, 10840, 11022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76368 dup_rows=43738 dup_rows_remote=38098 dedup_ratio=0.333694 router_tokens_per_expert=[509, 515, 471, 516, 474, 489, 499, 545, 571, 536, 514, 557, 485, 546, 505, 533, 483, 505, 514, 500, 504, 487, 508, 474, 521, 525, 526, 520, 496, 471, 536, 476, 507, 519, 505, 514, 494, 501, 532, 562, 476, 479, 510, 504, 497, 520, 537, 532, 537, 574, 527, 524, 490, 478, 510, 516, 549, 502, 535, 523, 487, 532, 503, 516, 454, 494, 554, 474, 501, 529, 512, 507, 526, 518, 503, 552, 498, 534, 537, 546, 502, 521, 537, 544, 479, 507, 524, 494, 509, 535, 473, 495, 489, 563, 511, 485, 492, 512, 458, 455, 521, 492, 507, 519, 523, 515, 516, 554, 541, 509, 512, 522, 494, 483, 561, 505, 489, 506, 523, 526, 495, 517, 518, 509, 542, 478, 491, 541] payload_rows_per_dst=[10940, 10788, 10952, 10966, 11002, 10896, 10874, 10916] dup_rows_per_dst=[5590, 5304, 5426, 5640, 5476, 5440, 5422, 5440] combine_rows_per_src=[10940, 10788, 10952, 10966, 11002, 10896, 10874, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626130944 dispatch_meta_bytes=457696 combine_payload_bytes=626130944 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626588640 ingress_bytes=852623360 link_bytes=852623360 token_bytes=1252719584 weight_movement_bytes=226492416 total_nvlink_bytes=1479212000 weight_to_token_ratio=0.180801 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081246436e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627376128 dispatch_meta_bytes=459384 combine_payload_bytes=627376128 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627835512 ingress_bytes=702873600 link_bytes=702873600 token_bytes=1255211640 weight_movement_bytes=75497472 total_nvlink_bytes=1330709112 weight_to_token_ratio=0.060147 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.913426617e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87574 payload_rows_remote=76604 dup_rows=43498 dup_rows_remote=37976 dedup_ratio=0.331863 router_tokens_per_expert=[554, 504, 499, 514, 469, 495, 520, 499, 526, 518, 555, 542, 515, 485, 468, 518, 515, 486, 489, 485, 499, 523, 506, 508, 508, 539, 517, 514, 502, 507, 506, 509, 482, 543, 524, 526, 537, 503, 498, 505, 478, 498, 493, 491, 516, 504, 551, 532, 535, 488, 460, 521, 527, 538, 526, 553, 508, 510, 521, 539, 559, 507, 506, 517, 521, 478, 523, 552, 490, 473, 499, 493, 432, 542, 508, 538, 437, 548, 460, 534, 563, 511, 569, 498, 593, 514, 501, 481, 550, 514, 499, 480, 488, 536, 521, 503, 490, 521, 505, 492, 500, 516, 511, 528, 529, 511, 481, 540, 510, 556, 530, 526, 506, 539, 535, 492, 504, 524, 505, 521, 482, 490, 492, 494, 526, 489, 538, 514] payload_rows_per_dst=[11014, 10836, 11024, 11140, 10758, 10968, 10970, 10864] dup_rows_per_dst=[5348, 5390, 5338, 5490, 5298, 5674, 5522, 5438] combine_rows_per_src=[11014, 10836, 11024, 11140, 10758, 10968, 10970, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=624525312 dispatch_meta_bytes=457248 combine_payload_bytes=624525312 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=624982560 ingress_bytes=775520256 link_bytes=775520256 token_bytes=1249507872 weight_movement_bytes=150994944 total_nvlink_bytes=1400502816 weight_to_token_ratio=0.120844 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.834688473e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87364 payload_rows_remote=76396 dup_rows=43708 dup_rows_remote=38296 dedup_ratio=0.333466 router_tokens_per_expert=[508, 540, 486, 534, 511, 524, 474, 496, 488, 460, 541, 516, 533, 505, 515, 488, 509, 539, 491, 504, 534, 501, 549, 550, 494, 539, 535, 526, 549, 483, 509, 529, 550, 520, 500, 493, 533, 500, 510, 486, 527, 561, 461, 519, 513, 539, 538, 547, 505, 487, 494, 517, 542, 503, 481, 467, 502, 501, 545, 509, 529, 528, 532, 545, 539, 504, 504, 462, 517, 531, 511, 508, 554, 508, 520, 543, 507, 528, 496, 517, 505, 518, 469, 442, 506, 509, 507, 481, 508, 498, 509, 539, 531, 481, 522, 530, 567, 481, 524, 494, 488, 521, 489, 509, 518, 530, 472, 487, 521, 477, 500, 520, 528, 459, 489, 493, 505, 537, 515, 539, 528, 510, 542, 530, 486, 511, 525, 493] payload_rows_per_dst=[10874, 10940, 10962, 10686, 11012, 10848, 11074, 10968] dup_rows_per_dst=[5320, 5742, 5632, 5032, 5366, 5426, 5778, 5412] combine_rows_per_src=[10874, 10940, 10962, 10686, 11012, 10848, 11074, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=459176 combine_payload_bytes=627326976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627786152 ingress_bytes=702824448 link_bytes=702824448 token_bytes=1255113128 weight_movement_bytes=75497472 total_nvlink_bytes=1330610600 weight_to_token_ratio=0.060152 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912803301e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625606656 dispatch_meta_bytes=457864 combine_payload_bytes=625606656 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626064520 ingress_bytes=852099072 link_bytes=852099072 token_bytes=1251671176 weight_movement_bytes=226492416 total_nvlink_bytes=1478163592 weight_to_token_ratio=0.180952 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080581565e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=458320 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627998288 ingress_bytes=703037440 link_bytes=703037440 token_bytes=1255538256 weight_movement_bytes=75497472 total_nvlink_bytes=1331035728 weight_to_token_ratio=0.060132 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915504339e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87556 payload_rows_remote=76594 dup_rows=43516 dup_rows_remote=38074 dedup_ratio=0.332001 router_tokens_per_expert=[550, 541, 529, 505, 550, 521, 496, 516, 514, 539, 512, 481, 523, 495, 501, 494, 529, 519, 493, 508, 530, 541, 475, 520, 487, 508, 515, 480, 526, 497, 514, 441, 524, 518, 500, 513, 518, 535, 493, 508, 509, 505, 469, 520, 518, 498, 522, 552, 512, 501, 502, 505, 504, 484, 527, 470, 508, 512, 560, 553, 506, 537, 519, 522, 503, 513, 483, 515, 516, 533, 484, 503, 549, 537, 537, 480, 534, 520, 534, 510, 518, 514, 503, 504, 494, 470, 498, 547, 477, 546, 527, 478, 487, 520, 515, 502, 542, 507, 537, 467, 497, 518, 520, 501, 498, 531, 490, 520, 542, 485, 528, 541, 507, 538, 495, 520, 473, 542, 527, 489, 523, 517, 493, 510, 521, 512, 504, 516] payload_rows_per_dst=[10976, 10838, 10962, 11022, 11016, 10850, 10950, 10942] dup_rows_per_dst=[5558, 5328, 5442, 5422, 5486, 5350, 5498, 5432] combine_rows_per_src=[10976, 10838, 10962, 11022, 11016, 10850, 10950, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458768 combine_payload_bytes=625836032 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=626294800 ingress_bytes=701333504 link_bytes=701333504 token_bytes=1252130832 weight_movement_bytes=75497472 total_nvlink_bytes=1327628304 weight_to_token_ratio=0.060295 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893896032e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87704 payload_rows_remote=76470 dup_rows=43368 dup_rows_remote=37906 dedup_ratio=0.330872 router_tokens_per_expert=[502, 521, 509, 477, 478, 514, 514, 499, 482, 511, 540, 528, 524, 520, 529, 522, 517, 496, 533, 469, 496, 526, 484, 525, 506, 517, 512, 501, 499, 540, 489, 512, 481, 530, 501, 473, 511, 539, 505, 545, 467, 520, 558, 463, 527, 484, 512, 540, 484, 512, 525, 505, 485, 535, 507, 545, 525, 545, 540, 506, 534, 528, 534, 470, 529, 537, 529, 509, 485, 510, 579, 469, 524, 525, 535, 550, 515, 558, 493, 501, 504, 540, 492, 491, 484, 477, 552, 503, 521, 493, 505, 543, 546, 514, 510, 519, 500, 495, 488, 531, 500, 498, 504, 470, 517, 519, 487, 502, 452, 518, 472, 503, 511, 530, 540, 523, 541, 488, 503, 511, 496, 505, 542, 539, 510, 558, 503, 510] payload_rows_per_dst=[10936, 10908, 10880, 11020, 11234, 11016, 10760, 10950] dup_rows_per_dst=[5404, 5336, 5432, 5540, 5462, 5372, 5152, 5670] combine_rows_per_src=[10936, 10908, 10880, 11020, 11234, 11016, 10760, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627458048 dispatch_meta_bytes=458672 combine_payload_bytes=627458048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627916720 ingress_bytes=702955520 link_bytes=702955520 token_bytes=1255374768 weight_movement_bytes=75497472 total_nvlink_bytes=1330872240 weight_to_token_ratio=0.060139 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914465478e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87632 payload_rows_remote=76618 dup_rows=43440 dup_rows_remote=38014 dedup_ratio=0.331421 router_tokens_per_expert=[531, 473, 543, 521, 510, 497, 531, 497, 520, 530, 480, 556, 512, 512, 487, 528, 509, 558, 532, 511, 512, 497, 498, 485, 515, 504, 513, 498, 518, 537, 540, 512, 563, 460, 524, 471, 519, 483, 494, 501, 516, 507, 487, 471, 549, 517, 520, 531, 479, 519, 529, 471, 520, 552, 523, 507, 472, 562, 526, 544, 468, 514, 479, 519, 536, 505, 520, 524, 522, 558, 517, 485, 466, 556, 505, 530, 515, 512, 508, 522, 469, 552, 473, 540, 524, 538, 510, 502, 503, 519, 504, 546, 489, 530, 512, 509, 528, 527, 494, 506, 527, 503, 547, 494, 496, 511, 508, 527, 462, 523, 517, 484, 489, 488, 512, 466, 484, 547, 532, 502, 520, 504, 485, 470, 551, 532, 538, 497] payload_rows_per_dst=[10988, 11042, 10820, 10912, 10998, 11014, 10960, 10898] dup_rows_per_dst=[5468, 5436, 5406, 5456, 5564, 5426, 5348, 5336] combine_rows_per_src=[10988, 11042, 10820, 10912, 10998, 11014, 10960, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87472 payload_rows_remote=76448 dup_rows=43600 dup_rows_remote=38362 dedup_ratio=0.332642 router_tokens_per_expert=[499, 518, 525, 532, 513, 506, 541, 498, 498, 534, 455, 484, 505, 488, 527, 508, 505, 502, 529, 515, 504, 505, 506, 514, 512, 553, 495, 522, 529, 516, 530, 488, 489, 502, 520, 507, 526, 535, 508, 504, 477, 511, 517, 506, 522, 542, 491, 487, 509, 523, 530, 508, 508, 528, 475, 535, 533, 479, 507, 512, 482, 479, 515, 518, 509, 555, 537, 512, 510, 496, 536, 477, 548, 527, 515, 530, 520, 541, 521, 516, 470, 503, 490, 498, 502, 486, 514, 506, 519, 532, 531, 510, 537, 527, 571, 510, 498, 522, 516, 526, 505, 499, 524, 516, 512, 519, 516, 514, 528, 498, 519, 489, 541, 524, 500, 529, 552, 476, 514, 478, 508, 486, 480, 528, 516, 518, 485, 503] payload_rows_per_dst=[11024, 11084, 10954, 10844, 10904, 10932, 10850, 10880] dup_rows_per_dst=[5238, 5822, 5578, 5258, 5340, 5480, 5552, 5332] combine_rows_per_src=[11024, 11084, 10954, 10844, 10904, 10932, 10850, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626442240 dispatch_meta_bytes=457504 combine_payload_bytes=626442240 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626899744 ingress_bytes=852934656 link_bytes=852934656 token_bytes=1253341984 weight_movement_bytes=226492416 total_nvlink_bytes=1479834400 weight_to_token_ratio=0.180711 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081641203e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87516 payload_rows_remote=76528 dup_rows=43556 dup_rows_remote=38050 dedup_ratio=0.332306 router_tokens_per_expert=[494, 503, 501, 578, 514, 524, 538, 500, 542, 529, 512, 504, 537, 495, 511, 474, 508, 476, 539, 510, 492, 504, 521, 495, 507, 544, 526, 531, 543, 534, 521, 496, 497, 508, 539, 501, 519, 483, 488, 512, 501, 525, 505, 528, 536, 517, 503, 510, 504, 490, 519, 506, 498, 505, 503, 497, 527, 528, 498, 444, 517, 507, 485, 522, 522, 512, 490, 486, 500, 478, 515, 501, 528, 524, 492, 524, 504, 504, 527, 475, 563, 536, 528, 484, 484, 504, 493, 464, 504, 538, 538, 544, 491, 507, 527, 477, 522, 520, 510, 516, 528, 517, 514, 551, 492, 509, 516, 539, 514, 500, 500, 523, 485, 527, 499, 473, 502, 557, 517, 505, 516, 532, 536, 527, 537, 522, 516, 525] payload_rows_per_dst=[11010, 10988, 10990, 10926, 10760, 10858, 11016, 10968] dup_rows_per_dst=[5502, 5506, 5354, 5174, 5404, 5506, 5526, 5584] combine_rows_per_src=[11010, 10988, 10990, 10926, 10760, 10858, 11016, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76636 dup_rows=43514 dup_rows_remote=38176 dedup_ratio=0.331985 router_tokens_per_expert=[518, 475, 506, 496, 501, 524, 475, 515, 532, 521, 529, 522, 509, 544, 527, 503, 524, 526, 540, 529, 468, 531, 523, 488, 524, 518, 516, 526, 504, 538, 494, 503, 517, 515, 563, 525, 542, 512, 483, 510, 514, 535, 536, 503, 523, 539, 501, 457, 520, 528, 515, 474, 471, 511, 490, 484, 545, 504, 499, 513, 523, 555, 505, 493, 516, 517, 495, 519, 528, 528, 463, 513, 485, 502, 516, 497, 498, 567, 525, 495, 491, 534, 509, 503, 548, 541, 514, 528, 491, 499, 501, 501, 511, 498, 514, 513, 497, 495, 516, 498, 507, 528, 516, 516, 482, 533, 485, 485, 517, 537, 520, 519, 522, 503, 490, 547, 496, 482, 528, 502, 555, 501, 509, 476, 490, 522, 537, 511] payload_rows_per_dst=[10940, 11008, 11018, 10922, 10922, 10930, 10908, 10910] dup_rows_per_dst=[5454, 5496, 5532, 5338, 5406, 5462, 5394, 5432] combine_rows_per_src=[10940, 11008, 11018, 10922, 10922, 10930, 10908, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76544 dup_rows=43534 dup_rows_remote=38070 dedup_ratio=0.332138 router_tokens_per_expert=[502, 509, 519, 488, 504, 493, 517, 519, 546, 531, 473, 499, 505, 494, 548, 482, 508, 471, 540, 546, 532, 503, 492, 506, 541, 512, 509, 523, 520, 495, 491, 507, 510, 515, 504, 522, 489, 544, 498, 493, 496, 496, 522, 538, 522, 510, 541, 520, 554, 494, 532, 485, 511, 495, 493, 520, 499, 536, 527, 495, 505, 518, 522, 520, 476, 498, 487, 525, 535, 493, 481, 512, 499, 498, 535, 503, 520, 495, 538, 520, 512, 511, 542, 506, 523, 529, 529, 561, 494, 525, 514, 518, 527, 516, 534, 525, 516, 465, 467, 515, 527, 520, 531, 512, 533, 527, 512, 507, 537, 494, 522, 544, 511, 495, 521, 487, 471, 521, 508, 519, 502, 525, 473, 547, 483, 517, 487, 508] payload_rows_per_dst=[10914, 10906, 10992, 10942, 10846, 11062, 10994, 10882] dup_rows_per_dst=[5344, 5486, 5448, 5470, 5384, 5670, 5464, 5268] combine_rows_per_src=[10914, 10906, 10992, 10942, 10846, 11062, 10994, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627654656 dispatch_meta_bytes=458528 combine_payload_bytes=627654656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628113184 ingress_bytes=703152128 link_bytes=703152128 token_bytes=1255767840 weight_movement_bytes=75497472 total_nvlink_bytes=1331265312 weight_to_token_ratio=0.060121 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916958744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626262016 dispatch_meta_bytes=459240 combine_payload_bytes=626262016 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=626721256 ingress_bytes=701759488 link_bytes=701759488 token_bytes=1252983272 weight_movement_bytes=75497472 total_nvlink_bytes=1328480744 weight_to_token_ratio=0.060254 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899298109e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87312 payload_rows_remote=76406 dup_rows=43760 dup_rows_remote=38166 dedup_ratio=0.333862 router_tokens_per_expert=[545, 533, 540, 500, 489, 522, 483, 501, 469, 511, 495, 513, 491, 540, 525, 473, 536, 478, 494, 540, 500, 483, 512, 510, 547, 504, 510, 530, 504, 504, 513, 512, 534, 516, 507, 534, 492, 492, 528, 496, 450, 533, 537, 516, 482, 520, 513, 520, 548, 516, 511, 549, 530, 519, 517, 529, 516, 503, 531, 484, 509, 479, 514, 479, 497, 510, 478, 493, 516, 518, 527, 537, 531, 537, 531, 553, 481, 533, 509, 496, 518, 542, 541, 517, 504, 476, 528, 518, 492, 529, 528, 502, 492, 553, 493, 473, 508, 496, 507, 502, 473, 535, 532, 480, 491, 501, 516, 488, 532, 490, 554, 520, 537, 489, 497, 517, 522, 542, 535, 506, 523, 474, 536, 522, 512, 508, 523, 504] payload_rows_per_dst=[10832, 10920, 10988, 10942, 10984, 10918, 10822, 10906] dup_rows_per_dst=[5428, 5434, 5352, 5526, 5510, 5494, 5422, 5594] combine_rows_per_src=[10832, 10920, 10988, 10942, 10984, 10918, 10822, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=458312 combine_payload_bytes=626917376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627375688 ingress_bytes=702414848 link_bytes=702414848 token_bytes=1254293064 weight_movement_bytes=75497472 total_nvlink_bytes=1329790536 weight_to_token_ratio=0.060191 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907608996e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=459248 combine_payload_bytes=627802112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628261360 ingress_bytes=703299584 link_bytes=703299584 token_bytes=1256063472 weight_movement_bytes=75497472 total_nvlink_bytes=1331560944 weight_to_token_ratio=0.060106 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918828694e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87410 payload_rows_remote=76534 dup_rows=43662 dup_rows_remote=38236 dedup_ratio=0.333115 router_tokens_per_expert=[521, 523, 524, 475, 462, 516, 500, 481, 542, 513, 521, 515, 502, 529, 531, 532, 502, 492, 520, 583, 496, 519, 505, 502, 498, 515, 449, 527, 539, 504, 503, 518, 506, 505, 493, 525, 542, 508, 505, 517, 476, 506, 519, 514, 497, 505, 526, 507, 473, 488, 462, 508, 558, 522, 498, 494, 541, 494, 535, 545, 538, 517, 532, 495, 546, 496, 465, 539, 546, 502, 505, 554, 460, 533, 537, 513, 511, 517, 498, 537, 522, 500, 492, 514, 487, 523, 523, 528, 491, 532, 494, 508, 515, 535, 491, 522, 482, 528, 530, 535, 505, 527, 501, 523, 523, 515, 536, 471, 481, 525, 509, 543, 471, 532, 547, 487, 534, 526, 473, 525, 508, 491, 510, 503, 508, 502, 517, 522] payload_rows_per_dst=[10902, 10852, 10876, 10968, 11040, 10922, 10972, 10878] dup_rows_per_dst=[5472, 5492, 5426, 5432, 5478, 5432, 5496, 5434] combine_rows_per_src=[10902, 10852, 10876, 10968, 11040, 10922, 10972, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627048448 dispatch_meta_bytes=458456 combine_payload_bytes=627048448 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627506904 ingress_bytes=778043392 link_bytes=778043392 token_bytes=1254555352 weight_movement_bytes=150994944 total_nvlink_bytes=1405550296 weight_to_token_ratio=0.120357 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866685389e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87608 payload_rows_remote=76574 dup_rows=43464 dup_rows_remote=38054 dedup_ratio=0.331604 router_tokens_per_expert=[523, 540, 488, 492, 494, 493, 559, 518, 496, 475, 496, 530, 503, 509, 480, 499, 506, 509, 520, 511, 489, 541, 497, 485, 517, 529, 502, 536, 480, 532, 523, 541, 487, 533, 485, 522, 479, 501, 505, 519, 485, 541, 502, 494, 511, 508, 482, 503, 506, 530, 500, 479, 477, 519, 484, 508, 510, 506, 521, 487, 531, 515, 528, 536, 496, 459, 516, 537, 496, 500, 522, 515, 534, 548, 501, 531, 513, 503, 512, 539, 541, 494, 549, 497, 532, 497, 512, 502, 497, 525, 533, 536, 519, 515, 564, 492, 479, 487, 547, 546, 517, 524, 521, 494, 521, 516, 518, 512, 558, 491, 511, 495, 523, 520, 539, 536, 508, 526, 528, 519, 510, 503, 511, 540, 528, 477, 475, 522] payload_rows_per_dst=[10912, 10926, 10806, 10896, 11034, 11098, 10964, 10972] dup_rows_per_dst=[5278, 5510, 5308, 5378, 5410, 5512, 5510, 5558] combine_rows_per_src=[10912, 10926, 10806, 10896, 11034, 11098, 10964, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625917952 dispatch_meta_bytes=458288 combine_payload_bytes=625917952 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626376240 ingress_bytes=776912896 link_bytes=776912896 token_bytes=1252294192 weight_movement_bytes=150994944 total_nvlink_bytes=1403289136 weight_to_token_ratio=0.120575 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.852349109e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626966528 dispatch_meta_bytes=459080 combine_payload_bytes=626966528 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627425608 ingress_bytes=702464000 link_bytes=702464000 token_bytes=1254392136 weight_movement_bytes=75497472 total_nvlink_bytes=1329889608 weight_to_token_ratio=0.060186 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908232313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87540 payload_rows_remote=76432 dup_rows=43532 dup_rows_remote=37926 dedup_ratio=0.332123 router_tokens_per_expert=[518, 546, 525, 509, 502, 507, 500, 531, 500, 496, 553, 495, 510, 521, 545, 549, 470, 504, 494, 500, 530, 470, 497, 501, 460, 525, 500, 521, 497, 509, 489, 519, 515, 524, 541, 553, 531, 512, 516, 493, 482, 523, 494, 505, 528, 483, 500, 519, 531, 511, 529, 505, 538, 500, 483, 506, 474, 537, 544, 537, 488, 524, 529, 476, 503, 496, 544, 530, 525, 542, 536, 492, 495, 507, 515, 475, 506, 491, 518, 518, 528, 497, 522, 526, 559, 503, 522, 543, 530, 571, 475, 484, 501, 579, 516, 501, 490, 507, 500, 529, 503, 485, 505, 485, 529, 507, 511, 510, 518, 535, 505, 485, 531, 521, 508, 516, 514, 492, 532, 468, 524, 498, 500, 479, 543, 518, 532, 482] payload_rows_per_dst=[11134, 10714, 10990, 10880, 10914, 11108, 10908, 10892] dup_rows_per_dst=[5480, 5258, 5448, 5544, 5472, 5606, 5300, 5424] combine_rows_per_src=[11134, 10714, 10990, 10880, 10914, 11108, 10908, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627294208 dispatch_meta_bytes=458512 combine_payload_bytes=627294208 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627752720 ingress_bytes=778289152 link_bytes=778289152 token_bytes=1255046928 weight_movement_bytes=150994944 total_nvlink_bytes=1406041872 weight_to_token_ratio=0.120310 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.869801972e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87414 payload_rows_remote=76688 dup_rows=43658 dup_rows_remote=38268 dedup_ratio=0.333084 router_tokens_per_expert=[508, 542, 545, 517, 536, 532, 513, 533, 505, 524, 498, 501, 521, 474, 499, 522, 523, 526, 484, 496, 477, 502, 523, 514, 518, 521, 493, 517, 468, 482, 508, 506, 536, 535, 502, 483, 503, 569, 520, 505, 499, 506, 491, 523, 495, 514, 523, 512, 485, 531, 503, 497, 528, 469, 495, 512, 494, 473, 510, 524, 517, 532, 507, 498, 491, 511, 544, 482, 530, 511, 508, 525, 499, 528, 488, 494, 520, 529, 485, 487, 542, 521, 516, 517, 536, 511, 509, 523, 540, 540, 529, 544, 532, 507, 525, 548, 488, 532, 524, 528, 480, 496, 551, 485, 521, 508, 528, 537, 489, 497, 503, 509, 483, 507, 496, 548, 553, 527, 536, 497, 506, 488, 525, 509, 491, 502, 523, 478] payload_rows_per_dst=[11028, 10726, 10872, 10778, 10894, 11178, 10974, 10964] dup_rows_per_dst=[5512, 5390, 5560, 5372, 5370, 5702, 5378, 5374] combine_rows_per_src=[11028, 10726, 10872, 10778, 10894, 11178, 10974, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87430 payload_rows_remote=76518 dup_rows=43642 dup_rows_remote=38084 dedup_ratio=0.332962 router_tokens_per_expert=[494, 517, 528, 503, 517, 517, 511, 533, 509, 504, 534, 512, 498, 514, 487, 512, 503, 492, 482, 547, 483, 515, 491, 557, 506, 503, 496, 521, 507, 506, 533, 473, 510, 524, 532, 516, 495, 524, 487, 496, 501, 525, 512, 512, 507, 480, 545, 484, 529, 512, 513, 518, 531, 459, 515, 505, 522, 503, 534, 521, 505, 524, 492, 545, 538, 531, 558, 498, 479, 546, 520, 530, 502, 510, 545, 510, 536, 537, 513, 514, 518, 508, 524, 515, 485, 559, 514, 501, 506, 473, 465, 496, 530, 478, 516, 498, 485, 531, 527, 513, 492, 510, 531, 528, 533, 550, 494, 486, 502, 500, 527, 497, 518, 506, 547, 494, 503, 485, 534, 529, 524, 509, 489, 500, 542, 535, 489, 490] payload_rows_per_dst=[10912, 11036, 11016, 11090, 10992, 10554, 10950, 10880] dup_rows_per_dst=[5558, 5540, 5528, 5644, 5516, 4940, 5426, 5490] combine_rows_per_src=[10912, 11036, 11016, 11090, 10992, 10554, 10950, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76686 dup_rows=43528 dup_rows_remote=38198 dedup_ratio=0.332092 router_tokens_per_expert=[518, 507, 503, 516, 502, 497, 517, 510, 540, 499, 536, 482, 514, 505, 520, 531, 576, 532, 472, 516, 480, 498, 539, 480, 505, 540, 538, 549, 502, 497, 485, 514, 516, 508, 498, 531, 562, 528, 528, 525, 517, 536, 506, 525, 495, 511, 513, 501, 486, 504, 486, 507, 511, 503, 523, 510, 502, 495, 497, 531, 521, 528, 503, 487, 519, 516, 518, 550, 524, 511, 512, 466, 499, 473, 481, 510, 532, 515, 537, 529, 529, 492, 503, 512, 506, 506, 534, 504, 556, 520, 489, 498, 525, 463, 516, 514, 524, 483, 549, 510, 485, 505, 497, 526, 531, 505, 546, 467, 473, 530, 511, 490, 500, 522, 500, 475, 492, 523, 494, 562, 518, 511, 497, 504, 514, 508, 534, 577] payload_rows_per_dst=[10972, 11038, 10972, 10858, 10950, 10854, 10926, 10974] dup_rows_per_dst=[5422, 5408, 5628, 5330, 5434, 5480, 5338, 5488] combine_rows_per_src=[10972, 11038, 10972, 10858, 10950, 10854, 10926, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626130944 dispatch_meta_bytes=457432 combine_payload_bytes=626130944 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626588376 ingress_bytes=928120832 link_bytes=928120832 token_bytes=1252719320 weight_movement_bytes=301989888 total_nvlink_bytes=1554709208 weight_to_token_ratio=0.241067 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176987858e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76514 dup_rows=43654 dup_rows_remote=38202 dedup_ratio=0.333054 router_tokens_per_expert=[482, 558, 505, 500, 534, 524, 497, 507, 531, 529, 506, 498, 511, 521, 535, 518, 513, 512, 518, 501, 533, 518, 492, 518, 517, 496, 500, 518, 504, 511, 487, 498, 525, 485, 511, 515, 493, 515, 536, 530, 560, 521, 542, 512, 505, 530, 494, 527, 485, 476, 508, 527, 525, 520, 483, 501, 532, 539, 506, 512, 485, 541, 513, 497, 502, 514, 570, 521, 573, 509, 497, 526, 470, 490, 493, 508, 513, 526, 509, 496, 531, 483, 530, 526, 479, 525, 521, 528, 501, 478, 491, 515, 522, 492, 493, 555, 524, 508, 530, 529, 506, 512, 484, 459, 528, 519, 511, 507, 521, 493, 518, 529, 481, 534, 521, 500, 544, 499, 505, 484, 500, 510, 474, 512, 489, 529, 503, 543] payload_rows_per_dst=[10994, 10912, 10964, 10906, 10930, 10928, 10904, 10880] dup_rows_per_dst=[5518, 5360, 5638, 5394, 5504, 5412, 5452, 5376] combine_rows_per_src=[10994, 10912, 10964, 10906, 10930, 10928, 10904, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87250 payload_rows_remote=76376 dup_rows=43822 dup_rows_remote=38308 dedup_ratio=0.334335 router_tokens_per_expert=[576, 520, 524, 506, 519, 523, 481, 559, 518, 522, 522, 474, 494, 509, 502, 555, 504, 517, 510, 528, 521, 510, 517, 470, 504, 484, 508, 511, 493, 529, 461, 490, 515, 540, 519, 496, 500, 545, 518, 526, 528, 529, 483, 540, 509, 508, 499, 513, 540, 505, 473, 533, 532, 509, 515, 522, 504, 543, 487, 495, 536, 521, 549, 521, 472, 542, 489, 502, 475, 521, 478, 483, 455, 524, 493, 520, 518, 499, 504, 497, 504, 538, 541, 496, 491, 550, 558, 525, 535, 495, 483, 470, 529, 500, 498, 517, 515, 499, 535, 526, 547, 507, 502, 537, 495, 511, 526, 467, 536, 546, 528, 524, 487, 526, 553, 563, 526, 507, 504, 508, 491, 518, 479, 499, 498, 483, 475, 502] payload_rows_per_dst=[10852, 10808, 10532, 11224, 10990, 10888, 11082, 10874] dup_rows_per_dst=[5434, 5316, 4838, 5984, 5562, 5654, 5520, 5514] combine_rows_per_src=[10852, 10808, 10532, 11224, 10990, 10888, 11082, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628228096 dispatch_meta_bytes=459824 combine_payload_bytes=628228096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628687920 ingress_bytes=703725568 link_bytes=703725568 token_bytes=1256916016 weight_movement_bytes=75497472 total_nvlink_bytes=1332413488 weight_to_token_ratio=0.060066 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924230771e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626835456 dispatch_meta_bytes=458408 combine_payload_bytes=626835456 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627293864 ingress_bytes=702332928 link_bytes=702332928 token_bytes=1254129320 weight_movement_bytes=75497472 total_nvlink_bytes=1329626792 weight_to_token_ratio=0.060199 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906570135e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628211712 dispatch_meta_bytes=459536 combine_payload_bytes=628211712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628671248 ingress_bytes=703709184 link_bytes=703709184 token_bytes=1256882960 weight_movement_bytes=75497472 total_nvlink_bytes=1332380432 weight_to_token_ratio=0.060067 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924022999e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87298 payload_rows_remote=76374 dup_rows=43774 dup_rows_remote=38264 dedup_ratio=0.333969 router_tokens_per_expert=[521, 511, 531, 508, 476, 502, 547, 505, 521, 527, 545, 520, 537, 519, 553, 504, 513, 529, 496, 515, 475, 524, 518, 522, 511, 510, 526, 527, 534, 536, 498, 473, 536, 537, 533, 496, 498, 475, 567, 526, 493, 489, 477, 511, 480, 480, 513, 545, 528, 497, 466, 502, 506, 497, 440, 500, 453, 504, 506, 508, 489, 476, 530, 516, 542, 523, 489, 511, 509, 468, 529, 486, 519, 505, 498, 528, 524, 559, 519, 508, 523, 536, 502, 515, 509, 536, 497, 519, 490, 473, 517, 523, 505, 537, 509, 516, 497, 518, 539, 492, 503, 510, 487, 507, 533, 523, 512, 519, 490, 489, 553, 523, 513, 505, 486, 513, 521, 539, 523, 513, 507, 498, 501, 572, 529, 526, 531, 532] payload_rows_per_dst=[11044, 10898, 10826, 10680, 10924, 10980, 10966, 10980] dup_rows_per_dst=[5610, 5516, 5486, 5156, 5510, 5434, 5424, 5638] combine_rows_per_src=[11044, 10898, 10826, 10680, 10924, 10980, 10966, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626802688 dispatch_meta_bytes=458864 combine_payload_bytes=626802688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627261552 ingress_bytes=702300160 link_bytes=702300160 token_bytes=1254064240 weight_movement_bytes=75497472 total_nvlink_bytes=1329561712 weight_to_token_ratio=0.060202 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906154591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87582 payload_rows_remote=76436 dup_rows=43490 dup_rows_remote=37984 dedup_ratio=0.331802 router_tokens_per_expert=[549, 499, 548, 517, 513, 460, 487, 517, 487, 505, 486, 509, 525, 486, 524, 513, 521, 507, 512, 508, 500, 493, 525, 537, 524, 514, 533, 533, 507, 517, 543, 497, 538, 517, 506, 564, 506, 539, 535, 510, 523, 527, 507, 513, 514, 480, 517, 530, 512, 538, 524, 509, 509, 548, 514, 543, 517, 462, 515, 482, 507, 528, 544, 514, 467, 493, 541, 515, 506, 515, 508, 472, 514, 507, 517, 514, 498, 505, 471, 509, 493, 479, 500, 492, 516, 513, 516, 496, 497, 536, 508, 522, 487, 531, 574, 498, 539, 496, 483, 466, 487, 531, 509, 469, 512, 529, 501, 540, 464, 510, 525, 513, 499, 522, 506, 514, 546, 529, 493, 492, 523, 519, 549, 541, 512, 485, 495, 539] payload_rows_per_dst=[10792, 11020, 11146, 10994, 10918, 10876, 10824, 11012] dup_rows_per_dst=[5458, 5522, 5506, 5538, 5186, 5440, 5324, 5516] combine_rows_per_src=[10792, 11020, 11146, 10994, 10918, 10876, 10824, 11012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625672192 dispatch_meta_bytes=458736 combine_payload_bytes=625672192 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626130928 ingress_bytes=776667136 link_bytes=776667136 token_bytes=1251803120 weight_movement_bytes=150994944 total_nvlink_bytes=1402798064 weight_to_token_ratio=0.120622 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.849232526e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625655808 dispatch_meta_bytes=458552 combine_payload_bytes=625655808 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626114360 ingress_bytes=701153280 link_bytes=701153280 token_bytes=1251770168 weight_movement_bytes=75497472 total_nvlink_bytes=1327267640 weight_to_token_ratio=0.060313 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.891610538e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87430 payload_rows_remote=76562 dup_rows=43642 dup_rows_remote=38348 dedup_ratio=0.332962 router_tokens_per_expert=[531, 491, 497, 538, 537, 498, 519, 490, 526, 479, 496, 457, 525, 492, 494, 538, 495, 473, 543, 512, 564, 489, 496, 510, 519, 545, 511, 500, 502, 569, 518, 538, 461, 488, 515, 524, 504, 523, 539, 557, 553, 507, 504, 507, 507, 518, 521, 520, 497, 509, 525, 488, 530, 516, 519, 515, 509, 489, 526, 492, 511, 548, 524, 498, 525, 542, 557, 514, 515, 497, 532, 534, 499, 522, 518, 511, 463, 516, 495, 487, 496, 500, 499, 494, 506, 508, 473, 497, 481, 513, 493, 498, 544, 521, 535, 523, 524, 518, 517, 506, 476, 509, 489, 514, 464, 519, 503, 506, 508, 475, 513, 503, 540, 483, 531, 508, 552, 553, 511, 527, 565, 540, 496, 532, 505, 508, 498, 499] payload_rows_per_dst=[10846, 10970, 10980, 10910, 11012, 10868, 10778, 11066] dup_rows_per_dst=[5370, 5598, 5516, 5482, 5442, 5294, 5310, 5630] combine_rows_per_src=[10846, 10970, 10980, 10910, 11012, 10868, 10778, 11066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626163712 dispatch_meta_bytes=457680 combine_payload_bytes=626163712 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626621392 ingress_bytes=852656128 link_bytes=852656128 token_bytes=1252785104 weight_movement_bytes=226492416 total_nvlink_bytes=1479277520 weight_to_token_ratio=0.180791 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081287991e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87388 payload_rows_remote=76540 dup_rows=43684 dup_rows_remote=38154 dedup_ratio=0.333282 router_tokens_per_expert=[509, 480, 512, 530, 549, 522, 526, 542, 527, 527, 460, 498, 477, 509, 498, 507, 498, 536, 477, 538, 528, 469, 538, 557, 501, 471, 507, 512, 519, 495, 510, 533, 520, 538, 484, 522, 485, 478, 524, 489, 490, 541, 488, 535, 524, 507, 503, 528, 502, 484, 553, 518, 508, 511, 517, 549, 484, 499, 503, 502, 521, 530, 519, 476, 514, 500, 491, 503, 525, 512, 535, 505, 526, 523, 470, 537, 533, 529, 525, 541, 504, 523, 521, 502, 498, 552, 524, 503, 506, 508, 508, 500, 523, 514, 532, 511, 474, 507, 539, 489, 487, 528, 563, 501, 483, 492, 509, 516, 520, 496, 526, 491, 515, 490, 506, 545, 488, 500, 479, 539, 505, 558, 510, 520, 549, 487, 527, 505] payload_rows_per_dst=[11006, 10848, 10878, 10878, 11002, 11006, 10896, 10874] dup_rows_per_dst=[5340, 5530, 5434, 5474, 5536, 5452, 5346, 5572] combine_rows_per_src=[11006, 10848, 10878, 10878, 11002, 11006, 10896, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87454 payload_rows_remote=76566 dup_rows=43618 dup_rows_remote=38286 dedup_ratio=0.332779 router_tokens_per_expert=[502, 525, 512, 522, 494, 503, 470, 514, 502, 501, 509, 520, 514, 517, 484, 521, 542, 510, 528, 497, 498, 505, 520, 542, 536, 529, 537, 492, 528, 547, 498, 486, 487, 550, 542, 496, 539, 533, 520, 511, 537, 502, 494, 533, 546, 489, 541, 484, 512, 497, 462, 514, 539, 481, 500, 528, 544, 542, 520, 504, 494, 528, 519, 549, 545, 510, 509, 521, 501, 500, 492, 561, 465, 525, 493, 524, 549, 495, 498, 535, 475, 516, 478, 492, 513, 501, 500, 492, 470, 493, 500, 550, 543, 512, 499, 508, 500, 540, 510, 530, 526, 512, 526, 481, 554, 514, 523, 509, 471, 498, 490, 481, 542, 517, 503, 481, 502, 539, 529, 484, 512, 532, 516, 485, 509, 517, 485, 511] payload_rows_per_dst=[10888, 10940, 10942, 11036, 11020, 10864, 11112, 10652] dup_rows_per_dst=[5332, 5650, 5666, 5430, 5426, 5220, 5760, 5134] combine_rows_per_src=[10888, 10940, 10942, 11036, 11020, 10864, 11112, 10652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87246 payload_rows_remote=76500 dup_rows=43826 dup_rows_remote=38384 dedup_ratio=0.334366 router_tokens_per_expert=[524, 520, 485, 515, 566, 473, 496, 512, 494, 517, 536, 517, 541, 516, 520, 534, 519, 515, 534, 551, 564, 514, 529, 521, 540, 506, 490, 512, 527, 497, 502, 522, 523, 550, 510, 480, 529, 518, 501, 524, 495, 489, 486, 475, 528, 485, 486, 527, 520, 497, 504, 538, 516, 497, 496, 513, 485, 505, 494, 503, 507, 496, 500, 523, 474, 521, 490, 519, 502, 534, 524, 538, 499, 481, 517, 507, 476, 515, 521, 536, 493, 463, 518, 505, 517, 524, 553, 504, 512, 498, 510, 533, 527, 485, 511, 487, 492, 515, 500, 507, 495, 517, 508, 503, 504, 512, 506, 473, 571, 537, 556, 526, 492, 505, 523, 544, 501, 545, 525, 539, 501, 489, 543, 492, 501, 493, 508, 510] payload_rows_per_dst=[11062, 11116, 10856, 10746, 10816, 10930, 10884, 10836] dup_rows_per_dst=[5470, 5570, 5356, 5442, 5492, 5350, 5560, 5586] combine_rows_per_src=[11062, 11116, 10856, 10746, 10816, 10930, 10884, 10836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=459640 combine_payload_bytes=627195904 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627655544 ingress_bytes=702693376 link_bytes=702693376 token_bytes=1254851448 weight_movement_bytes=75497472 total_nvlink_bytes=1330348920 weight_to_token_ratio=0.060164 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911141123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76672 dup_rows=43492 dup_rows_remote=38272 dedup_ratio=0.331818 router_tokens_per_expert=[546, 474, 536, 525, 505, 503, 505, 537, 524, 562, 534, 500, 481, 503, 536, 515, 518, 507, 535, 502, 530, 528, 499, 524, 503, 500, 523, 514, 539, 527, 486, 469, 502, 514, 528, 517, 495, 526, 507, 474, 537, 504, 481, 513, 482, 522, 521, 510, 498, 537, 514, 569, 539, 496, 558, 552, 528, 504, 501, 507, 495, 509, 527, 508, 515, 477, 502, 517, 509, 521, 490, 532, 507, 513, 511, 498, 518, 522, 512, 517, 503, 543, 500, 515, 465, 504, 529, 506, 527, 527, 518, 535, 531, 494, 489, 519, 523, 512, 482, 507, 498, 473, 505, 522, 499, 497, 494, 509, 526, 500, 499, 518, 527, 510, 511, 519, 502, 480, 485, 483, 505, 543, 494, 523, 497, 523, 497, 542] payload_rows_per_dst=[11004, 10964, 10924, 11068, 10880, 10936, 10908, 10896] dup_rows_per_dst=[5568, 5444, 5342, 5616, 5442, 5474, 5220, 5386] combine_rows_per_src=[11004, 10964, 10924, 11068, 10880, 10936, 10908, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87584 payload_rows_remote=76520 dup_rows=43488 dup_rows_remote=37776 dedup_ratio=0.331787 router_tokens_per_expert=[515, 541, 535, 501, 521, 578, 501, 503, 518, 505, 489, 513, 557, 506, 521, 504, 499, 481, 500, 519, 510, 555, 512, 504, 543, 517, 568, 508, 495, 535, 530, 497, 504, 502, 542, 533, 508, 521, 516, 571, 518, 514, 506, 524, 523, 492, 510, 495, 498, 500, 525, 502, 544, 499, 483, 513, 510, 506, 490, 486, 518, 515, 503, 494, 541, 491, 477, 526, 489, 517, 522, 472, 559, 497, 472, 543, 488, 509, 507, 529, 532, 515, 568, 496, 502, 469, 483, 499, 506, 483, 500, 500, 536, 505, 505, 473, 479, 487, 513, 537, 488, 510, 483, 530, 531, 520, 470, 536, 491, 512, 555, 534, 494, 501, 521, 524, 506, 536, 509, 517, 518, 498, 511, 505, 477, 519, 539, 528] payload_rows_per_dst=[10984, 10708, 11106, 10986, 10892, 10954, 10890, 11064] dup_rows_per_dst=[5422, 5088, 5624, 5332, 5438, 5328, 5544, 5712] combine_rows_per_src=[10984, 10708, 11106, 10986, 10892, 10954, 10890, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87126 payload_rows_remote=76266 dup_rows=43946 dup_rows_remote=38418 dedup_ratio=0.335281 router_tokens_per_expert=[491, 512, 526, 493, 493, 505, 501, 526, 463, 519, 498, 479, 494, 497, 516, 472, 519, 528, 529, 538, 532, 527, 555, 476, 505, 551, 494, 510, 469, 528, 494, 506, 501, 506, 491, 516, 524, 535, 516, 522, 523, 491, 535, 515, 515, 513, 513, 519, 539, 518, 535, 529, 543, 482, 528, 511, 498, 501, 504, 501, 543, 504, 520, 534, 480, 525, 493, 509, 495, 510, 495, 508, 515, 526, 528, 504, 530, 532, 487, 557, 494, 491, 505, 505, 526, 502, 498, 562, 541, 490, 543, 551, 517, 552, 520, 494, 515, 501, 486, 518, 549, 523, 486, 479, 515, 513, 518, 524, 529, 520, 490, 525, 500, 536, 494, 511, 459, 520, 488, 492, 521, 489, 537, 520, 477, 552, 478, 515] payload_rows_per_dst=[10702, 11008, 10918, 11008, 10860, 10918, 10906, 10806] dup_rows_per_dst=[5268, 5514, 5552, 5572, 5528, 5664, 5476, 5372] combine_rows_per_src=[10702, 11008, 10918, 11008, 10860, 10918, 10906, 10806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627228672 dispatch_meta_bytes=459408 combine_payload_bytes=627228672 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=627688080 ingress_bytes=929218560 link_bytes=929218560 token_bytes=1254916752 weight_movement_bytes=301989888 total_nvlink_bytes=1556906640 weight_to_token_ratio=0.240645 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178379931e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=458776 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=301989888 egress_bytes=627474456 ingress_bytes=1004503040 link_bytes=1004503040 token_bytes=1254490136 weight_movement_bytes=377487360 total_nvlink_bytes=1631977496 weight_to_token_ratio=0.300909 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.273851249e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626688000 dispatch_meta_bytes=459536 combine_payload_bytes=626688000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627147536 ingress_bytes=702185472 link_bytes=702185472 token_bytes=1253835536 weight_movement_bytes=75497472 total_nvlink_bytes=1329333008 weight_to_token_ratio=0.060213 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904700186e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87716 payload_rows_remote=76734 dup_rows=43356 dup_rows_remote=37972 dedup_ratio=0.330780 router_tokens_per_expert=[532, 507, 525, 533, 531, 528, 525, 536, 495, 525, 491, 465, 516, 494, 533, 519, 532, 506, 520, 512, 477, 522, 504, 547, 469, 502, 507, 486, 496, 511, 511, 520, 531, 504, 475, 504, 526, 491, 498, 533, 510, 505, 516, 542, 535, 510, 520, 483, 510, 488, 506, 484, 533, 511, 551, 520, 500, 496, 523, 495, 492, 489, 508, 528, 500, 508, 542, 547, 484, 457, 500, 545, 526, 524, 501, 547, 523, 508, 507, 502, 508, 538, 496, 499, 495, 525, 559, 475, 503, 530, 552, 493, 531, 494, 506, 510, 515, 526, 510, 504, 518, 537, 535, 512, 502, 539, 507, 484, 556, 514, 516, 519, 526, 484, 506, 525, 506, 513, 513, 486, 483, 485, 504, 504, 544, 525, 500, 509] payload_rows_per_dst=[10984, 10870, 10982, 10966, 11050, 10906, 11076, 10882] dup_rows_per_dst=[5526, 5374, 5384, 5302, 5392, 5522, 5512, 5344] combine_rows_per_src=[10984, 10870, 10982, 10966, 11050, 10906, 11076, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628097024 dispatch_meta_bytes=459776 combine_payload_bytes=628097024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628556800 ingress_bytes=703594496 link_bytes=703594496 token_bytes=1256653824 weight_movement_bytes=75497472 total_nvlink_bytes=1332151296 weight_to_token_ratio=0.060078 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922568593e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=457184 combine_payload_bytes=626851840 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627309024 ingress_bytes=702349312 link_bytes=702349312 token_bytes=1254160864 weight_movement_bytes=75497472 total_nvlink_bytes=1329658336 weight_to_token_ratio=0.060198 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906777908e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=624771072 dispatch_meta_bytes=458736 combine_payload_bytes=624771072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625229808 ingress_bytes=700268544 link_bytes=700268544 token_bytes=1250000880 weight_movement_bytes=75497472 total_nvlink_bytes=1325498352 weight_to_token_ratio=0.060398 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.880390840e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76658 dup_rows=43534 dup_rows_remote=38090 dedup_ratio=0.332138 router_tokens_per_expert=[535, 518, 500, 516, 496, 565, 501, 466, 489, 532, 517, 535, 528, 490, 495, 533, 511, 501, 580, 506, 554, 498, 489, 552, 509, 509, 528, 475, 568, 556, 528, 497, 509, 536, 529, 512, 516, 507, 548, 520, 524, 527, 532, 517, 471, 538, 475, 484, 498, 556, 504, 522, 537, 494, 518, 515, 515, 498, 481, 494, 499, 547, 501, 490, 509, 499, 523, 522, 493, 519, 519, 521, 467, 520, 483, 515, 511, 537, 507, 510, 516, 504, 526, 503, 530, 461, 525, 519, 520, 555, 478, 525, 484, 521, 492, 503, 527, 516, 527, 539, 494, 507, 506, 517, 479, 516, 494, 485, 514, 499, 541, 521, 495, 521, 501, 485, 476, 472, 557, 522, 505, 532, 524, 510, 479, 496, 491, 480] payload_rows_per_dst=[10978, 11096, 10988, 10936, 10912, 10880, 10980, 10768] dup_rows_per_dst=[5454, 5626, 5502, 5402, 5398, 5444, 5384, 5324] combine_rows_per_src=[10978, 11096, 10988, 10936, 10912, 10880, 10980, 10768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628604928 dispatch_meta_bytes=458824 combine_payload_bytes=628604928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629063752 ingress_bytes=704102400 link_bytes=704102400 token_bytes=1257668680 weight_movement_bytes=75497472 total_nvlink_bytes=1333166152 weight_to_token_ratio=0.060030 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.929009531e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627982336 dispatch_meta_bytes=458992 combine_payload_bytes=627982336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628441328 ingress_bytes=703479808 link_bytes=703479808 token_bytes=1256423664 weight_movement_bytes=75497472 total_nvlink_bytes=1331921136 weight_to_token_ratio=0.060089 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921114188e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87320 payload_rows_remote=76486 dup_rows=43752 dup_rows_remote=38282 dedup_ratio=0.333801 router_tokens_per_expert=[483, 535, 486, 512, 494, 528, 516, 535, 508, 531, 558, 478, 494, 464, 509, 521, 489, 537, 509, 516, 478, 504, 475, 539, 534, 494, 508, 526, 524, 510, 489, 526, 531, 535, 491, 537, 487, 483, 487, 543, 502, 528, 493, 474, 516, 495, 490, 521, 490, 517, 493, 517, 512, 517, 547, 527, 464, 500, 513, 530, 546, 529, 498, 472, 510, 472, 520, 516, 514, 504, 539, 507, 563, 529, 528, 484, 512, 511, 485, 523, 486, 533, 513, 512, 518, 488, 516, 489, 515, 526, 524, 492, 534, 480, 537, 491, 503, 525, 536, 497, 483, 498, 561, 536, 492, 516, 544, 518, 528, 524, 479, 505, 495, 515, 529, 541, 538, 522, 497, 512, 540, 501, 560, 519, 492, 515, 567, 482] payload_rows_per_dst=[10834, 10976, 10796, 10838, 10878, 10934, 10930, 11134] dup_rows_per_dst=[5470, 5340, 5430, 5506, 5556, 5374, 5560, 5516] combine_rows_per_src=[10834, 10976, 10796, 10838, 10878, 10934, 10930, 11134] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87524 payload_rows_remote=76430 dup_rows=43548 dup_rows_remote=38114 dedup_ratio=0.332245 router_tokens_per_expert=[480, 497, 518, 482, 522, 496, 526, 515, 485, 542, 531, 498, 534, 519, 532, 523, 524, 535, 518, 499, 513, 510, 530, 521, 483, 534, 519, 494, 480, 563, 497, 544, 499, 499, 530, 502, 531, 493, 532, 541, 562, 518, 522, 493, 512, 507, 524, 512, 537, 505, 522, 529, 526, 476, 520, 497, 541, 522, 504, 522, 487, 519, 493, 518, 486, 513, 514, 515, 495, 519, 512, 555, 481, 495, 509, 511, 512, 481, 525, 478, 549, 526, 497, 500, 489, 530, 526, 523, 492, 506, 494, 541, 508, 498, 532, 473, 520, 498, 531, 516, 492, 512, 487, 526, 550, 524, 483, 505, 512, 499, 504, 446, 474, 508, 562, 481, 511, 508, 503, 538, 532, 508, 516, 466, 535, 513, 513, 519] payload_rows_per_dst=[10948, 11094, 10960, 10940, 10788, 10994, 10850, 10950] dup_rows_per_dst=[5452, 5434, 5594, 5496, 5414, 5374, 5360, 5424] combine_rows_per_src=[10948, 11094, 10960, 10940, 10788, 10994, 10850, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87400 payload_rows_remote=76328 dup_rows=43672 dup_rows_remote=38330 dedup_ratio=0.333191 router_tokens_per_expert=[517, 495, 485, 494, 520, 502, 484, 474, 523, 542, 532, 505, 527, 525, 490, 499, 495, 538, 548, 511, 499, 473, 512, 522, 539, 468, 533, 531, 513, 516, 517, 500, 504, 513, 510, 514, 576, 496, 526, 525, 521, 492, 531, 549, 514, 493, 489, 513, 532, 499, 490, 484, 478, 497, 543, 554, 537, 561, 535, 503, 502, 467, 523, 502, 519, 510, 500, 512, 499, 491, 516, 520, 521, 472, 527, 501, 531, 493, 500, 512, 538, 520, 465, 526, 526, 523, 491, 514, 501, 496, 485, 478, 547, 519, 475, 515, 581, 506, 491, 491, 476, 537, 520, 481, 506, 513, 509, 511, 504, 545, 505, 515, 501, 532, 543, 549, 507, 518, 484, 511, 486, 544, 512, 546, 509, 514, 517, 527] payload_rows_per_dst=[10916, 11020, 10920, 11072, 10802, 10806, 10886, 10978] dup_rows_per_dst=[5312, 5410, 5612, 5342, 5446, 5432, 5496, 5622] combine_rows_per_src=[10916, 11020, 10920, 11072, 10802, 10806, 10886, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87426 payload_rows_remote=76530 dup_rows=43646 dup_rows_remote=38030 dedup_ratio=0.332993 router_tokens_per_expert=[484, 517, 564, 505, 511, 556, 486, 482, 476, 506, 500, 545, 465, 515, 539, 533, 482, 506, 499, 493, 505, 478, 538, 483, 505, 541, 496, 494, 504, 505, 494, 538, 519, 492, 504, 506, 507, 519, 484, 536, 507, 535, 451, 516, 527, 511, 488, 517, 501, 501, 510, 538, 484, 496, 513, 548, 510, 531, 524, 524, 532, 503, 515, 539, 520, 491, 502, 553, 490, 479, 551, 533, 533, 512, 510, 502, 520, 508, 518, 461, 528, 536, 546, 547, 486, 521, 513, 510, 512, 543, 522, 528, 523, 504, 503, 530, 521, 517, 536, 514, 493, 528, 540, 495, 515, 505, 512, 535, 533, 500, 442, 500, 502, 507, 484, 491, 523, 472, 523, 537, 511, 509, 523, 516, 540, 497, 525, 522] payload_rows_per_dst=[10890, 10802, 10832, 11130, 10836, 11112, 10896, 10928] dup_rows_per_dst=[5478, 5320, 5266, 5408, 5530, 5592, 5616, 5436] combine_rows_per_src=[10890, 10802, 10832, 11130, 10836, 11112, 10896, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87642 payload_rows_remote=76588 dup_rows=43430 dup_rows_remote=37962 dedup_ratio=0.331345 router_tokens_per_expert=[516, 537, 492, 521, 528, 523, 513, 530, 521, 497, 555, 531, 504, 508, 481, 480, 510, 495, 571, 492, 494, 524, 497, 531, 484, 466, 492, 488, 524, 508, 511, 549, 544, 532, 536, 545, 540, 512, 463, 494, 549, 517, 528, 514, 541, 515, 485, 512, 501, 546, 539, 539, 517, 467, 490, 500, 498, 484, 541, 525, 527, 511, 551, 488, 513, 510, 532, 500, 502, 483, 489, 489, 505, 498, 507, 558, 521, 490, 518, 501, 536, 493, 506, 510, 507, 497, 492, 547, 525, 492, 516, 514, 548, 519, 501, 506, 505, 489, 504, 528, 511, 518, 535, 539, 519, 535, 514, 491, 489, 482, 487, 468, 517, 467, 498, 495, 506, 480, 521, 509, 559, 507, 477, 530, 553, 564, 482, 508] payload_rows_per_dst=[10876, 10952, 10550, 10706, 11130, 11000, 11374, 11054] dup_rows_per_dst=[5358, 5320, 5024, 5200, 5690, 5436, 5934, 5468] combine_rows_per_src=[10876, 10952, 10550, 10706, 11130, 11000, 11374, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87316 payload_rows_remote=76254 dup_rows=43756 dup_rows_remote=38302 dedup_ratio=0.333832 router_tokens_per_expert=[525, 511, 538, 497, 537, 485, 501, 524, 498, 503, 524, 507, 526, 499, 531, 495, 566, 540, 505, 495, 527, 526, 503, 521, 506, 516, 532, 497, 497, 522, 538, 477, 526, 556, 521, 529, 479, 515, 509, 511, 495, 514, 541, 495, 504, 506, 548, 516, 518, 511, 510, 495, 544, 506, 507, 521, 524, 506, 525, 482, 488, 502, 539, 490, 536, 526, 503, 530, 510, 507, 494, 545, 503, 506, 540, 522, 468, 501, 539, 528, 499, 485, 496, 496, 496, 503, 560, 458, 530, 529, 534, 510, 526, 527, 488, 552, 497, 501, 490, 534, 514, 482, 483, 532, 529, 513, 494, 496, 491, 508, 517, 475, 556, 494, 474, 515, 513, 513, 486, 500, 538, 501, 501, 494, 528, 487, 532, 499] payload_rows_per_dst=[10844, 10962, 10998, 10902, 11062, 10938, 10742, 10868] dup_rows_per_dst=[5558, 5574, 5532, 5434, 5454, 5440, 5370, 5394] combine_rows_per_src=[10844, 10962, 10998, 10902, 11062, 10938, 10742, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626573312 dispatch_meta_bytes=459072 combine_payload_bytes=626573312 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627032384 ingress_bytes=853065728 link_bytes=853065728 token_bytes=1253605696 weight_movement_bytes=226492416 total_nvlink_bytes=1480098112 weight_to_token_ratio=0.180673 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081807421e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626114560 dispatch_meta_bytes=458176 combine_payload_bytes=626114560 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626572736 ingress_bytes=701612032 link_bytes=701612032 token_bytes=1252687296 weight_movement_bytes=75497472 total_nvlink_bytes=1328184768 weight_to_token_ratio=0.060268 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897428159e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625278976 dispatch_meta_bytes=458632 combine_payload_bytes=625278976 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625737608 ingress_bytes=851771392 link_bytes=851771392 token_bytes=1251016584 weight_movement_bytes=226492416 total_nvlink_bytes=1477509000 weight_to_token_ratio=0.181047 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080166021e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87734 payload_rows_remote=76650 dup_rows=43338 dup_rows_remote=37684 dedup_ratio=0.330643 router_tokens_per_expert=[511, 534, 516, 529, 579, 495, 510, 528, 523, 488, 483, 511, 529, 488, 495, 524, 499, 510, 461, 509, 491, 512, 487, 500, 544, 568, 506, 523, 518, 536, 476, 547, 486, 515, 491, 515, 533, 514, 530, 548, 536, 536, 533, 564, 528, 523, 541, 476, 541, 508, 530, 532, 480, 555, 481, 509, 528, 539, 501, 522, 519, 502, 533, 511, 481, 525, 480, 488, 518, 523, 480, 500, 496, 526, 536, 476, 515, 475, 512, 516, 516, 488, 514, 539, 555, 511, 498, 513, 523, 486, 505, 522, 511, 516, 548, 492, 490, 494, 501, 488, 521, 462, 480, 485, 455, 563, 494, 480, 509, 503, 502, 531, 503, 552, 534, 534, 521, 537, 501, 498, 500, 502, 524, 545, 529, 463, 474, 487] payload_rows_per_dst=[11010, 10980, 11084, 11036, 10816, 11006, 10772, 11030] dup_rows_per_dst=[5476, 5394, 5654, 5546, 5278, 5468, 5144, 5378] combine_rows_per_src=[11010, 10980, 11084, 11036, 10816, 11006, 10772, 11030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626933760 dispatch_meta_bytes=458240 combine_payload_bytes=626933760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627392000 ingress_bytes=702431232 link_bytes=702431232 token_bytes=1254325760 weight_movement_bytes=75497472 total_nvlink_bytes=1329823232 weight_to_token_ratio=0.060190 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907816768e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627408896 dispatch_meta_bytes=458200 combine_payload_bytes=627408896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627867096 ingress_bytes=702906368 link_bytes=702906368 token_bytes=1255275992 weight_movement_bytes=75497472 total_nvlink_bytes=1330773464 weight_to_token_ratio=0.060144 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.913842162e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624672768 dispatch_meta_bytes=458224 combine_payload_bytes=624672768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625130992 ingress_bytes=700170240 link_bytes=700170240 token_bytes=1249803760 weight_movement_bytes=75497472 total_nvlink_bytes=1325301232 weight_to_token_ratio=0.060407 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.879144207e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87222 payload_rows_remote=76228 dup_rows=43850 dup_rows_remote=38216 dedup_ratio=0.334549 router_tokens_per_expert=[488, 516, 522, 516, 528, 476, 500, 525, 501, 489, 505, 531, 522, 490, 500, 498, 550, 520, 532, 520, 513, 484, 510, 544, 516, 499, 514, 501, 477, 512, 520, 534, 486, 501, 544, 496, 533, 516, 514, 512, 481, 516, 507, 502, 513, 518, 527, 492, 513, 484, 507, 478, 513, 560, 514, 536, 516, 523, 534, 509, 536, 535, 531, 507, 496, 472, 520, 519, 455, 530, 526, 512, 521, 541, 474, 509, 522, 483, 537, 519, 499, 500, 498, 567, 560, 513, 531, 526, 510, 524, 519, 499, 491, 529, 535, 513, 522, 518, 536, 510, 481, 544, 545, 492, 497, 514, 545, 474, 500, 478, 519, 505, 557, 497, 488, 517, 538, 492, 477, 508, 502, 507, 526, 487, 500, 501, 514, 488] payload_rows_per_dst=[10812, 10958, 10926, 10940, 10840, 10994, 10952, 10800] dup_rows_per_dst=[5402, 5534, 5390, 5652, 5432, 5634, 5408, 5398] combine_rows_per_src=[10812, 10958, 10926, 10940, 10840, 10994, 10952, 10800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=457336 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628374136 ingress_bytes=778911744 link_bytes=778911744 token_bytes=1256290936 weight_movement_bytes=150994944 total_nvlink_bytes=1407285880 weight_to_token_ratio=0.120191 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.877697315e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624459776 dispatch_meta_bytes=457776 combine_payload_bytes=624459776 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624917552 ingress_bytes=775454720 link_bytes=775454720 token_bytes=1249377328 weight_movement_bytes=150994944 total_nvlink_bytes=1400372272 weight_to_token_ratio=0.120856 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.833857385e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87238 payload_rows_remote=76138 dup_rows=43834 dup_rows_remote=38194 dedup_ratio=0.334427 router_tokens_per_expert=[539, 527, 502, 537, 514, 483, 497, 547, 547, 510, 539, 509, 546, 522, 519, 532, 506, 553, 484, 507, 548, 495, 468, 514, 527, 509, 510, 497, 491, 509, 515, 467, 483, 499, 483, 483, 510, 495, 508, 472, 489, 523, 506, 524, 545, 534, 549, 537, 528, 471, 494, 496, 509, 544, 545, 521, 519, 508, 500, 492, 525, 526, 523, 492, 518, 515, 547, 480, 557, 490, 502, 517, 532, 535, 513, 515, 558, 476, 517, 505, 512, 508, 520, 525, 500, 496, 520, 518, 525, 497, 505, 531, 486, 502, 529, 474, 479, 510, 488, 490, 522, 510, 521, 540, 469, 502, 546, 513, 515, 495, 502, 534, 484, 519, 542, 495, 483, 539, 528, 475, 508, 507, 520, 504, 542, 529, 503, 494] payload_rows_per_dst=[11100, 10822, 11072, 10824, 10940, 10592, 10850, 11038] dup_rows_per_dst=[5640, 5496, 5658, 5444, 5614, 5254, 5422, 5306] combine_rows_per_src=[11100, 10822, 11072, 10824, 10940, 10592, 10850, 11038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87582 payload_rows_remote=76676 dup_rows=43490 dup_rows_remote=38298 dedup_ratio=0.331802 router_tokens_per_expert=[483, 495, 541, 560, 482, 524, 508, 522, 512, 480, 515, 513, 529, 516, 496, 549, 470, 493, 509, 533, 487, 501, 505, 507, 520, 493, 497, 534, 464, 481, 564, 491, 519, 520, 525, 498, 544, 501, 499, 513, 508, 538, 552, 521, 518, 510, 502, 489, 501, 498, 485, 547, 524, 525, 518, 499, 502, 493, 505, 508, 476, 532, 527, 516, 475, 487, 489, 482, 539, 520, 518, 513, 519, 493, 562, 540, 500, 526, 536, 546, 511, 552, 500, 519, 537, 507, 522, 552, 517, 538, 571, 521, 518, 492, 524, 510, 496, 518, 499, 524, 528, 495, 491, 483, 508, 487, 494, 507, 484, 491, 530, 493, 494, 515, 508, 482, 495, 507, 537, 516, 506, 496, 529, 500, 513, 542, 523, 522] payload_rows_per_dst=[11010, 10906, 10974, 10886, 10928, 11130, 10860, 10888] dup_rows_per_dst=[5440, 5192, 5540, 5426, 5562, 5652, 5196, 5482] combine_rows_per_src=[11010, 10906, 10974, 10886, 10928, 11130, 10860, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87390 payload_rows_remote=76550 dup_rows=43682 dup_rows_remote=38270 dedup_ratio=0.333267 router_tokens_per_expert=[467, 497, 494, 496, 506, 532, 553, 513, 537, 507, 515, 520, 480, 515, 472, 528, 483, 526, 500, 530, 516, 527, 508, 506, 489, 481, 506, 481, 491, 522, 480, 490, 508, 540, 533, 497, 515, 507, 499, 543, 505, 483, 533, 491, 533, 507, 547, 522, 489, 485, 511, 526, 521, 528, 470, 529, 524, 505, 540, 486, 497, 527, 479, 509, 514, 506, 509, 487, 521, 507, 512, 520, 488, 534, 470, 499, 514, 502, 491, 495, 515, 527, 556, 547, 542, 526, 541, 519, 487, 488, 516, 501, 474, 526, 519, 530, 513, 520, 541, 511, 498, 496, 530, 531, 506, 511, 493, 545, 459, 532, 492, 552, 502, 500, 542, 482, 515, 518, 510, 538, 514, 545, 502, 534, 503, 574, 530, 557] payload_rows_per_dst=[10836, 10778, 10990, 10840, 10794, 11048, 10952, 11152] dup_rows_per_dst=[5428, 5294, 5536, 5412, 5344, 5580, 5508, 5580] combine_rows_per_src=[10836, 10778, 10990, 10840, 10794, 11048, 10952, 11152] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87440 payload_rows_remote=76650 dup_rows=43632 dup_rows_remote=38260 dedup_ratio=0.332886 router_tokens_per_expert=[526, 524, 566, 532, 522, 506, 503, 535, 511, 478, 505, 475, 511, 505, 526, 542, 507, 558, 490, 541, 509, 514, 496, 507, 496, 550, 489, 558, 508, 495, 529, 516, 493, 492, 507, 515, 466, 513, 531, 552, 517, 498, 515, 510, 534, 569, 511, 506, 555, 526, 554, 492, 538, 519, 517, 493, 560, 488, 512, 543, 534, 550, 474, 507, 495, 522, 515, 450, 557, 528, 514, 476, 489, 512, 484, 515, 509, 519, 507, 489, 504, 503, 489, 513, 506, 508, 513, 475, 503, 462, 525, 537, 488, 509, 510, 493, 519, 533, 518, 491, 505, 533, 473, 531, 500, 528, 522, 487, 494, 484, 497, 510, 545, 475, 510, 503, 492, 451, 568, 490, 564, 520, 533, 525, 482, 493, 537, 483] payload_rows_per_dst=[10992, 10934, 10954, 11108, 10790, 10862, 10914, 10886] dup_rows_per_dst=[5542, 5592, 5504, 5616, 5372, 5214, 5336, 5456] combine_rows_per_src=[10992, 10934, 10954, 11108, 10790, 10862, 10914, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87416 payload_rows_remote=76314 dup_rows=43656 dup_rows_remote=38150 dedup_ratio=0.333069 router_tokens_per_expert=[523, 498, 491, 517, 488, 495, 497, 533, 500, 531, 501, 525, 504, 509, 515, 507, 490, 513, 467, 489, 537, 476, 534, 516, 489, 511, 538, 515, 490, 506, 497, 538, 513, 536, 486, 540, 508, 540, 487, 539, 479, 481, 527, 497, 507, 527, 508, 519, 515, 568, 544, 493, 555, 523, 491, 540, 467, 492, 507, 515, 479, 501, 504, 502, 546, 470, 521, 525, 487, 505, 470, 521, 507, 504, 528, 535, 517, 534, 483, 529, 577, 529, 456, 490, 500, 501, 510, 520, 488, 501, 515, 532, 508, 515, 530, 502, 538, 528, 542, 500, 527, 470, 514, 521, 481, 531, 566, 493, 502, 516, 554, 521, 509, 533, 531, 520, 524, 547, 528, 493, 518, 542, 467, 487, 523, 522, 484, 518] payload_rows_per_dst=[10856, 10886, 10826, 10902, 10914, 10884, 11102, 11046] dup_rows_per_dst=[5412, 5326, 5562, 5490, 5450, 5464, 5506, 5446] combine_rows_per_src=[10856, 10886, 10826, 10902, 10914, 10884, 11102, 11046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76634 dup_rows=43590 dup_rows_remote=38296 dedup_ratio=0.332565 router_tokens_per_expert=[521, 493, 509, 496, 496, 469, 542, 505, 537, 544, 508, 502, 555, 522, 515, 514, 536, 536, 515, 500, 530, 508, 495, 495, 573, 546, 511, 483, 494, 532, 503, 486, 475, 457, 488, 504, 510, 507, 509, 519, 525, 495, 495, 517, 489, 521, 525, 477, 496, 494, 517, 558, 507, 499, 517, 526, 518, 503, 514, 519, 551, 503, 515, 519, 484, 494, 586, 515, 522, 507, 483, 525, 521, 533, 488, 547, 477, 518, 514, 494, 552, 485, 554, 500, 516, 511, 517, 527, 465, 505, 510, 573, 507, 523, 506, 532, 530, 475, 531, 499, 525, 532, 501, 499, 495, 513, 496, 550, 537, 504, 517, 537, 532, 501, 514, 510, 465, 528, 526, 486, 496, 527, 486, 468, 536, 481, 497, 511] payload_rows_per_dst=[10832, 10978, 10972, 10976, 10924, 10950, 11002, 10848] dup_rows_per_dst=[5026, 5508, 5436, 5384, 5644, 5616, 5682, 5294] combine_rows_per_src=[10832, 10978, 10972, 10976, 10924, 10950, 11002, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=623722496 dispatch_meta_bytes=457328 combine_payload_bytes=623722496 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=624179824 ingress_bytes=850214912 link_bytes=850214912 token_bytes=1247902320 weight_movement_bytes=226492416 total_nvlink_bytes=1474394736 weight_to_token_ratio=0.181499 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078192185e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627097600 dispatch_meta_bytes=459280 combine_payload_bytes=627097600 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627556880 ingress_bytes=853590016 link_bytes=853590016 token_bytes=1254654480 weight_movement_bytes=226492416 total_nvlink_bytes=1481146896 weight_to_token_ratio=0.180522 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082472292e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=459896 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628589688 ingress_bytes=703627264 link_bytes=703627264 token_bytes=1256719480 weight_movement_bytes=75497472 total_nvlink_bytes=1332216952 weight_to_token_ratio=0.060075 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922984138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=459640 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628376440 ingress_bytes=703414272 link_bytes=703414272 token_bytes=1256293240 weight_movement_bytes=75497472 total_nvlink_bytes=1331790712 weight_to_token_ratio=0.060095 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920283099e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87354 payload_rows_remote=76552 dup_rows=43718 dup_rows_remote=38428 dedup_ratio=0.333542 router_tokens_per_expert=[534, 526, 518, 525, 483, 543, 537, 527, 483, 487, 513, 543, 514, 506, 512, 530, 486, 549, 526, 504, 502, 523, 483, 541, 536, 546, 515, 528, 520, 487, 475, 546, 503, 545, 517, 479, 526, 513, 499, 458, 482, 570, 514, 478, 490, 546, 466, 506, 528, 506, 552, 493, 545, 471, 489, 526, 548, 500, 546, 496, 540, 491, 546, 512, 489, 525, 519, 543, 499, 513, 524, 499, 514, 521, 486, 502, 506, 511, 557, 541, 503, 520, 535, 509, 471, 464, 498, 553, 516, 476, 490, 473, 506, 481, 559, 492, 490, 507, 515, 519, 523, 517, 505, 498, 538, 519, 482, 478, 481, 487, 543, 520, 519, 503, 495, 509, 564, 535, 501, 541, 491, 502, 537, 477, 518, 509, 462, 527] payload_rows_per_dst=[11044, 10968, 10716, 11038, 10914, 10802, 10926, 10946] dup_rows_per_dst=[5518, 5566, 5468, 5540, 5584, 5290, 5318, 5434] combine_rows_per_src=[11044, 10968, 10716, 11038, 10914, 10802, 10926, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625164288 dispatch_meta_bytes=457856 combine_payload_bytes=625164288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625622144 ingress_bytes=700661760 link_bytes=700661760 token_bytes=1250786432 weight_movement_bytes=75497472 total_nvlink_bytes=1326283904 weight_to_token_ratio=0.060360 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.885377373e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87322 payload_rows_remote=76464 dup_rows=43750 dup_rows_remote=38244 dedup_ratio=0.333786 router_tokens_per_expert=[538, 556, 529, 455, 533, 543, 519, 487, 479, 578, 503, 523, 484, 536, 533, 511, 491, 509, 512, 529, 528, 524, 516, 514, 491, 488, 485, 525, 519, 502, 558, 528, 508, 513, 505, 536, 490, 507, 514, 472, 493, 492, 549, 544, 518, 535, 486, 520, 516, 523, 487, 501, 516, 513, 490, 545, 522, 537, 507, 492, 568, 497, 503, 513, 497, 498, 536, 510, 506, 452, 520, 551, 540, 471, 472, 498, 556, 492, 505, 503, 545, 509, 509, 508, 493, 504, 528, 501, 464, 520, 513, 519, 480, 508, 548, 526, 526, 477, 513, 558, 519, 501, 502, 525, 502, 483, 484, 484, 511, 529, 524, 525, 518, 488, 545, 507, 470, 519, 543, 485, 518, 509, 510, 506, 496, 504, 490, 545] payload_rows_per_dst=[11036, 10992, 10858, 10896, 10742, 10998, 10834, 10966] dup_rows_per_dst=[5578, 5446, 5506, 5564, 5472, 5352, 5492, 5340] combine_rows_per_src=[11036, 10992, 10858, 10896, 10742, 10998, 10834, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=459720 combine_payload_bytes=627785728 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628245448 ingress_bytes=778780672 link_bytes=778780672 token_bytes=1256031176 weight_movement_bytes=150994944 total_nvlink_bytes=1407026120 weight_to_token_ratio=0.120216 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876035138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627113984 dispatch_meta_bytes=459920 combine_payload_bytes=627113984 prefetch_bytes=0 reduce_bytes=0 egress_bytes=627573904 ingress_bytes=627113984 link_bytes=627573904 token_bytes=1254687888 weight_movement_bytes=0 total_nvlink_bytes=1254687888 weight_to_token_ratio=0.000000 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=7.958520480e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626393088 dispatch_meta_bytes=458832 combine_payload_bytes=626393088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626851920 ingress_bytes=701890560 link_bytes=701890560 token_bytes=1253245008 weight_movement_bytes=75497472 total_nvlink_bytes=1328742480 weight_to_token_ratio=0.060242 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900960286e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87232 payload_rows_remote=76314 dup_rows=43840 dup_rows_remote=38236 dedup_ratio=0.334473 router_tokens_per_expert=[498, 540, 527, 525, 514, 553, 518, 489, 501, 509, 496, 544, 463, 511, 538, 535, 522, 528, 440, 510, 549, 481, 488, 494, 489, 531, 502, 514, 485, 529, 477, 547, 500, 507, 516, 505, 501, 510, 554, 494, 560, 527, 531, 457, 496, 452, 483, 522, 528, 474, 513, 523, 482, 548, 508, 507, 532, 548, 528, 511, 523, 516, 528, 479, 513, 510, 535, 494, 504, 556, 528, 475, 525, 551, 547, 522, 551, 512, 510, 486, 529, 512, 515, 551, 512, 520, 513, 482, 493, 456, 522, 505, 533, 487, 502, 483, 510, 527, 541, 509, 552, 495, 529, 533, 480, 487, 537, 497, 524, 525, 485, 491, 504, 462, 520, 521, 487, 510, 542, 504, 490, 508, 536, 535, 488, 517, 546, 500] payload_rows_per_dst=[10918, 10986, 10798, 10714, 11094, 10862, 10958, 10902] dup_rows_per_dst=[5604, 5720, 5432, 5280, 5544, 5368, 5486, 5406] combine_rows_per_src=[10918, 10986, 10798, 10714, 11094, 10862, 10958, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76540 dup_rows=43492 dup_rows_remote=37968 dedup_ratio=0.331818 router_tokens_per_expert=[500, 517, 496, 512, 551, 506, 492, 509, 534, 533, 514, 499, 483, 528, 568, 543, 499, 508, 538, 539, 497, 522, 506, 549, 518, 474, 489, 515, 537, 449, 515, 510, 487, 487, 517, 542, 512, 513, 506, 501, 456, 510, 510, 494, 530, 539, 559, 485, 513, 518, 498, 525, 510, 510, 501, 501, 537, 518, 532, 511, 490, 530, 534, 554, 544, 520, 480, 513, 460, 477, 489, 486, 493, 557, 505, 524, 506, 473, 507, 522, 504, 534, 497, 501, 533, 492, 534, 473, 522, 494, 494, 531, 558, 503, 465, 523, 567, 494, 535, 523, 524, 474, 506, 493, 484, 505, 478, 501, 529, 519, 480, 558, 504, 536, 529, 507, 485, 495, 477, 529, 503, 517, 506, 533, 568, 506, 545, 532] payload_rows_per_dst=[11040, 10862, 10834, 11040, 10870, 10942, 10928, 11064] dup_rows_per_dst=[5530, 5468, 5462, 5524, 5242, 5374, 5412, 5480] combine_rows_per_src=[11040, 10862, 10834, 11040, 10870, 10942, 10928, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87290 payload_rows_remote=76264 dup_rows=43782 dup_rows_remote=38248 dedup_ratio=0.334030 router_tokens_per_expert=[460, 540, 525, 500, 524, 517, 456, 508, 525, 461, 543, 522, 495, 539, 485, 503, 506, 492, 536, 491, 542, 505, 534, 517, 460, 556, 490, 527, 516, 514, 547, 513, 479, 495, 518, 474, 517, 506, 535, 506, 496, 521, 537, 524, 532, 504, 498, 512, 518, 529, 504, 528, 543, 526, 533, 481, 524, 494, 540, 516, 488, 512, 509, 538, 528, 528, 508, 515, 519, 531, 516, 509, 546, 555, 491, 524, 521, 527, 508, 454, 515, 528, 507, 533, 519, 514, 531, 495, 485, 537, 502, 436, 511, 503, 459, 519, 461, 491, 514, 480, 522, 517, 526, 502, 501, 492, 511, 538, 560, 516, 525, 540, 525, 534, 535, 504, 518, 502, 504, 524, 548, 496, 494, 518, 505, 505, 496, 472] payload_rows_per_dst=[10866, 10984, 10876, 10912, 11026, 10800, 10874, 10952] dup_rows_per_dst=[5340, 5508, 5432, 5654, 5534, 5388, 5518, 5408] combine_rows_per_src=[10866, 10984, 10876, 10912, 11026, 10800, 10874, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87412 payload_rows_remote=76462 dup_rows=43660 dup_rows_remote=38118 dedup_ratio=0.333099 router_tokens_per_expert=[516, 522, 497, 538, 499, 495, 545, 500, 528, 519, 495, 486, 551, 506, 516, 493, 538, 474, 521, 481, 533, 502, 524, 517, 488, 521, 554, 531, 496, 576, 496, 494, 542, 485, 484, 534, 521, 499, 496, 492, 501, 550, 509, 509, 515, 507, 528, 521, 507, 507, 530, 487, 510, 475, 467, 490, 545, 517, 537, 490, 524, 474, 509, 508, 536, 491, 517, 495, 499, 475, 500, 519, 519, 531, 556, 537, 522, 493, 532, 505, 494, 500, 511, 517, 524, 519, 537, 528, 485, 512, 526, 506, 458, 568, 515, 501, 538, 517, 565, 493, 520, 529, 474, 500, 506, 463, 520, 542, 496, 498, 501, 522, 484, 509, 493, 513, 512, 529, 516, 507, 542, 509, 535, 524, 511, 487, 515, 516] payload_rows_per_dst=[10930, 10950, 10910, 10844, 10938, 10930, 10956, 10954] dup_rows_per_dst=[5482, 5542, 5476, 5310, 5516, 5472, 5412, 5450] combine_rows_per_src=[10930, 10950, 10910, 10844, 10938, 10930, 10956, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87522 payload_rows_remote=76434 dup_rows=43550 dup_rows_remote=38030 dedup_ratio=0.332260 router_tokens_per_expert=[481, 519, 511, 533, 524, 476, 506, 502, 508, 489, 499, 518, 501, 493, 495, 545, 504, 480, 499, 546, 550, 503, 524, 487, 508, 514, 479, 529, 543, 520, 528, 493, 540, 523, 509, 481, 536, 512, 506, 509, 468, 522, 517, 517, 498, 504, 551, 474, 495, 488, 523, 496, 511, 482, 563, 519, 499, 479, 530, 482, 521, 538, 552, 507, 547, 533, 526, 515, 464, 500, 527, 501, 535, 473, 555, 479, 537, 520, 517, 523, 518, 471, 510, 532, 499, 539, 522, 495, 517, 510, 494, 502, 513, 543, 504, 522, 519, 482, 509, 495, 550, 536, 547, 542, 542, 476, 545, 494, 521, 525, 510, 511, 530, 501, 517, 524, 508, 458, 513, 486, 503, 498, 510, 551, 513, 509, 537, 472] payload_rows_per_dst=[10866, 10862, 10880, 10848, 11028, 10932, 11088, 11018] dup_rows_per_dst=[5334, 5552, 5454, 5522, 5476, 5450, 5520, 5242] combine_rows_per_src=[10866, 10862, 10880, 10848, 11028, 10932, 11088, 11018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76652 dup_rows=43454 dup_rows_remote=37996 dedup_ratio=0.331528 router_tokens_per_expert=[528, 487, 530, 520, 488, 524, 507, 505, 531, 544, 530, 482, 487, 536, 492, 523, 528, 537, 514, 509, 521, 512, 550, 525, 499, 471, 526, 498, 523, 502, 503, 520, 490, 497, 526, 491, 553, 550, 532, 503, 510, 508, 538, 486, 490, 519, 525, 505, 488, 531, 504, 491, 528, 501, 447, 518, 536, 506, 525, 505, 502, 529, 477, 530, 502, 511, 513, 538, 468, 501, 514, 531, 537, 512, 507, 506, 515, 509, 503, 517, 487, 482, 459, 501, 509, 526, 497, 535, 511, 490, 481, 573, 549, 494, 524, 518, 534, 521, 543, 496, 513, 509, 489, 529, 502, 554, 506, 523, 524, 501, 541, 510, 504, 503, 519, 483, 486, 475, 486, 509, 528, 500, 509, 552, 492, 534, 496, 552] payload_rows_per_dst=[10962, 10880, 10996, 10818, 10778, 11166, 11052, 10966] dup_rows_per_dst=[5466, 5306, 5452, 5278, 5256, 5602, 5636, 5458] combine_rows_per_src=[10962, 10880, 10996, 10818, 10778, 11166, 11052, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625164288 dispatch_meta_bytes=458200 combine_payload_bytes=625164288 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=625622488 ingress_bytes=851656704 link_bytes=851656704 token_bytes=1250786776 weight_movement_bytes=226492416 total_nvlink_bytes=1477279192 weight_to_token_ratio=0.181080 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080020580e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=458032 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627473712 ingress_bytes=853508096 link_bytes=853508096 token_bytes=1254489392 weight_movement_bytes=226492416 total_nvlink_bytes=1480981808 weight_to_token_ratio=0.180546 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082368406e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=624754688 dispatch_meta_bytes=458048 combine_payload_bytes=624754688 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625212736 ingress_bytes=775749632 link_bytes=775749632 token_bytes=1249967424 weight_movement_bytes=150994944 total_nvlink_bytes=1400962368 weight_to_token_ratio=0.120799 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.837597284e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626376704 dispatch_meta_bytes=458320 combine_payload_bytes=626376704 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=626835024 ingress_bytes=928366592 link_bytes=928366592 token_bytes=1253211728 weight_movement_bytes=301989888 total_nvlink_bytes=1555201616 weight_to_token_ratio=0.240973 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177299516e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76498 dup_rows=43654 dup_rows_remote=38154 dedup_ratio=0.333054 router_tokens_per_expert=[502, 519, 515, 539, 536, 496, 498, 516, 520, 498, 504, 506, 497, 507, 506, 525, 529, 496, 490, 505, 494, 547, 515, 510, 511, 484, 498, 469, 513, 515, 540, 529, 540, 505, 530, 528, 503, 506, 487, 479, 496, 507, 506, 481, 503, 537, 516, 554, 516, 541, 492, 487, 520, 497, 524, 500, 518, 505, 527, 504, 508, 522, 514, 481, 481, 519, 502, 531, 543, 543, 525, 513, 568, 508, 550, 517, 479, 517, 490, 529, 515, 501, 463, 513, 539, 515, 538, 504, 528, 525, 506, 528, 505, 503, 497, 530, 513, 523, 493, 500, 503, 512, 489, 545, 477, 520, 507, 495, 509, 492, 514, 454, 508, 486, 496, 538, 509, 543, 532, 524, 528, 514, 527, 513, 503, 529, 528, 524] payload_rows_per_dst=[10936, 10884, 10946, 10882, 11044, 10920, 10792, 11014] dup_rows_per_dst=[5432, 5406, 5410, 5430, 5586, 5500, 5300, 5590] combine_rows_per_src=[10936, 10884, 10946, 10882, 11044, 10920, 10792, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626147328 dispatch_meta_bytes=457856 combine_payload_bytes=626147328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626605184 ingress_bytes=701644800 link_bytes=701644800 token_bytes=1252752512 weight_movement_bytes=75497472 total_nvlink_bytes=1328249984 weight_to_token_ratio=0.060265 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897843704e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627933184 dispatch_meta_bytes=458592 combine_payload_bytes=627933184 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628391776 ingress_bytes=703430656 link_bytes=703430656 token_bytes=1256324960 weight_movement_bytes=75497472 total_nvlink_bytes=1331822432 weight_to_token_ratio=0.060094 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920490872e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87408 payload_rows_remote=76364 dup_rows=43664 dup_rows_remote=38112 dedup_ratio=0.333130 router_tokens_per_expert=[519, 504, 536, 522, 495, 518, 497, 485, 503, 515, 506, 484, 527, 526, 495, 523, 511, 541, 532, 491, 518, 528, 509, 515, 475, 508, 460, 551, 501, 515, 557, 514, 508, 495, 515, 511, 560, 521, 489, 529, 523, 550, 524, 502, 511, 500, 529, 531, 563, 511, 478, 543, 495, 537, 542, 493, 451, 521, 539, 501, 543, 542, 509, 529, 516, 557, 501, 514, 487, 564, 508, 487, 511, 497, 528, 526, 484, 516, 507, 540, 480, 502, 449, 520, 468, 526, 512, 525, 518, 495, 503, 474, 483, 534, 509, 490, 511, 526, 510, 503, 530, 494, 466, 527, 515, 489, 514, 517, 518, 504, 492, 537, 528, 490, 495, 537, 502, 537, 523, 497, 509, 515, 530, 515, 532, 491, 476, 499] payload_rows_per_dst=[10918, 10976, 11044, 11100, 10906, 10736, 10868, 10860] dup_rows_per_dst=[5392, 5476, 5552, 5494, 5580, 5240, 5438, 5492] combine_rows_per_src=[10918, 10976, 11044, 11100, 10906, 10736, 10868, 10860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626671616 dispatch_meta_bytes=458608 combine_payload_bytes=626671616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627130224 ingress_bytes=702169088 link_bytes=702169088 token_bytes=1253801840 weight_movement_bytes=75497472 total_nvlink_bytes=1329299312 weight_to_token_ratio=0.060215 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904492414e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87740 payload_rows_remote=76714 dup_rows=43332 dup_rows_remote=37780 dedup_ratio=0.330597 router_tokens_per_expert=[505, 501, 479, 508, 505, 510, 535, 553, 561, 518, 506, 502, 534, 527, 510, 508, 492, 530, 528, 497, 546, 511, 530, 523, 515, 498, 517, 492, 508, 482, 524, 515, 526, 521, 492, 534, 525, 565, 517, 484, 490, 494, 521, 514, 498, 510, 494, 489, 497, 517, 518, 526, 536, 526, 532, 494, 471, 511, 491, 494, 527, 538, 513, 544, 527, 513, 472, 468, 503, 488, 532, 525, 521, 517, 540, 515, 505, 516, 481, 523, 539, 509, 507, 518, 506, 500, 543, 551, 532, 472, 513, 550, 474, 526, 527, 543, 502, 527, 498, 545, 490, 503, 496, 494, 566, 508, 488, 507, 472, 525, 501, 517, 477, 509, 499, 524, 499, 546, 492, 503, 497, 511, 474, 509, 480, 475, 542, 525] payload_rows_per_dst=[11026, 11010, 10984, 10976, 10876, 11052, 10896, 10920] dup_rows_per_dst=[5552, 5406, 5364, 5440, 5416, 5568, 5382, 5204] combine_rows_per_src=[11026, 11010, 10984, 10976, 10876, 11052, 10896, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625573888 dispatch_meta_bytes=457904 combine_payload_bytes=625573888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626031792 ingress_bytes=701071360 link_bytes=701071360 token_bytes=1251605680 weight_movement_bytes=75497472 total_nvlink_bytes=1327103152 weight_to_token_ratio=0.060320 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890571677e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87584 payload_rows_remote=76604 dup_rows=43488 dup_rows_remote=38154 dedup_ratio=0.331787 router_tokens_per_expert=[502, 528, 506, 493, 490, 507, 508, 552, 530, 521, 506, 555, 497, 477, 531, 505, 544, 496, 525, 505, 522, 515, 484, 492, 486, 518, 510, 497, 537, 499, 540, 569, 494, 538, 519, 483, 520, 504, 484, 529, 500, 525, 525, 501, 542, 498, 540, 495, 522, 490, 513, 511, 479, 538, 492, 523, 529, 508, 496, 536, 525, 506, 498, 491, 541, 478, 517, 515, 511, 496, 526, 485, 536, 518, 543, 495, 527, 507, 519, 521, 508, 487, 529, 511, 520, 542, 502, 554, 501, 497, 513, 502, 489, 536, 536, 483, 465, 468, 520, 499, 458, 506, 531, 518, 497, 531, 500, 523, 486, 514, 524, 511, 487, 512, 507, 523, 531, 513, 487, 482, 538, 519, 496, 523, 525, 572, 534, 490] payload_rows_per_dst=[11050, 10960, 10856, 10980, 11016, 10902, 10882, 10938] dup_rows_per_dst=[5366, 5518, 5538, 5334, 5454, 5518, 5220, 5540] combine_rows_per_src=[11050, 10960, 10856, 10980, 11016, 10902, 10882, 10938] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87582 payload_rows_remote=76646 dup_rows=43490 dup_rows_remote=38106 dedup_ratio=0.331802 router_tokens_per_expert=[489, 501, 478, 528, 509, 497, 462, 498, 501, 497, 511, 477, 525, 502, 532, 509, 499, 494, 539, 512, 561, 474, 551, 520, 534, 549, 537, 528, 510, 516, 535, 528, 497, 498, 497, 529, 573, 534, 536, 491, 505, 512, 521, 518, 520, 504, 495, 506, 522, 514, 519, 502, 510, 529, 508, 518, 526, 483, 502, 532, 497, 562, 527, 527, 512, 480, 461, 504, 519, 505, 534, 532, 520, 506, 560, 518, 509, 499, 504, 497, 511, 522, 458, 471, 526, 470, 489, 492, 542, 498, 490, 506, 501, 519, 514, 528, 490, 497, 528, 535, 541, 541, 491, 495, 516, 509, 534, 494, 507, 523, 543, 501, 513, 476, 483, 552, 487, 520, 518, 479, 524, 523, 511, 495, 535, 530, 518, 513] payload_rows_per_dst=[10796, 11098, 10986, 10968, 10936, 10812, 10980, 11006] dup_rows_per_dst=[5236, 5676, 5486, 5588, 5384, 5262, 5510, 5348] combine_rows_per_src=[10796, 11098, 10986, 10968, 10936, 10812, 10980, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87692 payload_rows_remote=76720 dup_rows=43380 dup_rows_remote=37858 dedup_ratio=0.330963 router_tokens_per_expert=[515, 506, 506, 513, 483, 523, 511, 521, 516, 512, 508, 511, 512, 486, 527, 533, 537, 478, 525, 511, 462, 527, 520, 517, 551, 529, 515, 532, 493, 550, 481, 519, 471, 479, 520, 501, 515, 511, 490, 507, 512, 490, 502, 501, 517, 498, 497, 515, 515, 529, 537, 539, 512, 507, 511, 481, 525, 514, 542, 487, 525, 495, 505, 513, 505, 528, 510, 525, 519, 555, 515, 508, 535, 505, 486, 500, 495, 470, 495, 500, 487, 555, 524, 518, 515, 544, 525, 542, 508, 549, 516, 504, 510, 477, 507, 492, 484, 502, 514, 544, 535, 509, 468, 533, 509, 524, 506, 508, 542, 510, 522, 506, 511, 497, 500, 487, 502, 540, 517, 502, 509, 515, 523, 505, 536, 526, 551, 482] payload_rows_per_dst=[10964, 10972, 10856, 11048, 10992, 10982, 10950, 10928] dup_rows_per_dst=[5402, 5522, 5196, 5426, 5310, 5564, 5482, 5478] combine_rows_per_src=[10964, 10972, 10856, 11048, 10992, 10982, 10950, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76484 dup_rows=43590 dup_rows_remote=38202 dedup_ratio=0.332565 router_tokens_per_expert=[541, 461, 508, 505, 470, 500, 515, 488, 525, 504, 511, 466, 517, 540, 501, 537, 495, 495, 508, 492, 515, 506, 547, 490, 500, 536, 520, 532, 487, 538, 566, 523, 536, 524, 501, 485, 512, 502, 515, 529, 466, 497, 528, 538, 498, 481, 572, 454, 528, 553, 545, 497, 534, 531, 505, 516, 495, 504, 498, 481, 514, 518, 522, 539, 498, 506, 504, 529, 479, 537, 499, 495, 509, 493, 522, 508, 525, 583, 490, 463, 470, 518, 522, 528, 507, 510, 522, 555, 515, 510, 524, 513, 509, 514, 536, 514, 518, 536, 532, 511, 508, 516, 494, 485, 527, 533, 506, 508, 521, 468, 515, 515, 514, 520, 529, 513, 498, 526, 503, 517, 494, 523, 507, 474, 519, 520, 486, 536] payload_rows_per_dst=[10952, 10946, 10766, 11006, 10872, 11020, 10998, 10922] dup_rows_per_dst=[5226, 5554, 5510, 5554, 5408, 5514, 5388, 5436] combine_rows_per_src=[10952, 10946, 10766, 11006, 10872, 11020, 10998, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628441088 dispatch_meta_bytes=457976 combine_payload_bytes=628441088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628899064 ingress_bytes=703938560 link_bytes=703938560 token_bytes=1257340152 weight_movement_bytes=75497472 total_nvlink_bytes=1332837624 weight_to_token_ratio=0.060045 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926931809e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87356 payload_rows_remote=76374 dup_rows=43716 dup_rows_remote=38170 dedup_ratio=0.333527 router_tokens_per_expert=[515, 484, 515, 556, 486, 484, 508, 553, 524, 546, 514, 519, 525, 551, 511, 567, 495, 528, 492, 501, 500, 483, 508, 460, 500, 511, 518, 508, 473, 528, 517, 531, 494, 472, 514, 477, 522, 476, 537, 525, 503, 516, 519, 500, 551, 521, 581, 486, 472, 556, 533, 497, 477, 503, 476, 560, 503, 518, 500, 517, 521, 532, 481, 524, 496, 519, 530, 466, 483, 501, 489, 510, 495, 558, 543, 521, 499, 520, 507, 522, 449, 507, 512, 515, 529, 508, 523, 502, 505, 532, 499, 481, 516, 524, 535, 523, 538, 502, 505, 516, 506, 513, 543, 519, 491, 477, 519, 506, 509, 502, 486, 546, 530, 539, 495, 511, 534, 474, 538, 526, 499, 482, 527, 501, 515, 513, 566, 514] payload_rows_per_dst=[11040, 10820, 10884, 10918, 10884, 10952, 10876, 10982] dup_rows_per_dst=[5676, 5222, 5478, 5386, 5434, 5430, 5544, 5546] combine_rows_per_src=[11040, 10820, 10884, 10918, 10884, 10952, 10876, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=459032 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627999000 ingress_bytes=778534912 link_bytes=778534912 token_bytes=1255538968 weight_movement_bytes=150994944 total_nvlink_bytes=1406533912 weight_to_token_ratio=0.120263 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872918555e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627884032 dispatch_meta_bytes=459008 combine_payload_bytes=627884032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628343040 ingress_bytes=703381504 link_bytes=703381504 token_bytes=1256227072 weight_movement_bytes=75497472 total_nvlink_bytes=1331724544 weight_to_token_ratio=0.060099 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919867555e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628490240 dispatch_meta_bytes=458312 combine_payload_bytes=628490240 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628948552 ingress_bytes=854982656 link_bytes=854982656 token_bytes=1257438792 weight_movement_bytes=226492416 total_nvlink_bytes=1483931208 weight_to_token_ratio=0.180122 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084238356e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87444 payload_rows_remote=76426 dup_rows=43628 dup_rows_remote=38160 dedup_ratio=0.332855 router_tokens_per_expert=[510, 507, 490, 510, 502, 499, 491, 506, 517, 544, 506, 527, 521, 533, 565, 519, 530, 507, 510, 548, 525, 513, 535, 510, 484, 507, 519, 519, 498, 537, 475, 533, 478, 515, 492, 507, 534, 525, 519, 528, 511, 529, 527, 547, 510, 488, 533, 515, 513, 504, 518, 526, 527, 511, 479, 528, 496, 486, 536, 506, 493, 517, 502, 494, 502, 541, 506, 489, 523, 499, 496, 486, 550, 515, 520, 517, 492, 523, 528, 504, 534, 521, 554, 504, 523, 504, 529, 495, 519, 522, 496, 526, 497, 524, 483, 512, 490, 535, 505, 498, 522, 522, 489, 509, 519, 492, 490, 535, 481, 504, 500, 528, 515, 508, 551, 482, 506, 496, 489, 506, 540, 537, 485, 509, 500, 474, 506, 488] payload_rows_per_dst=[10938, 10968, 10926, 10858, 10926, 11018, 10930, 10880] dup_rows_per_dst=[5556, 5532, 5590, 5414, 5456, 5468, 5308, 5304] combine_rows_per_src=[10938, 10968, 10926, 10858, 10926, 11018, 10930, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=458744 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627015672 ingress_bytes=702054400 link_bytes=702054400 token_bytes=1253572600 weight_movement_bytes=75497472 total_nvlink_bytes=1329070072 weight_to_token_ratio=0.060226 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903038008e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625655808 dispatch_meta_bytes=458176 combine_payload_bytes=625655808 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626113984 ingress_bytes=852148224 link_bytes=852148224 token_bytes=1251769792 weight_movement_bytes=226492416 total_nvlink_bytes=1478262208 weight_to_token_ratio=0.180938 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080643897e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87654 payload_rows_remote=76674 dup_rows=43418 dup_rows_remote=37940 dedup_ratio=0.331253 router_tokens_per_expert=[502, 512, 525, 504, 472, 480, 536, 500, 512, 493, 539, 489, 493, 540, 515, 496, 493, 517, 483, 541, 484, 508, 494, 492, 516, 497, 537, 512, 555, 529, 545, 535, 500, 588, 472, 500, 521, 527, 503, 497, 540, 502, 526, 516, 523, 501, 501, 512, 510, 488, 504, 532, 494, 499, 519, 510, 544, 453, 529, 511, 541, 501, 490, 540, 469, 528, 524, 487, 511, 512, 497, 551, 504, 509, 501, 519, 570, 493, 525, 555, 473, 511, 548, 529, 532, 545, 478, 499, 516, 470, 508, 533, 491, 498, 494, 534, 518, 508, 525, 527, 497, 512, 530, 536, 541, 527, 488, 493, 555, 509, 514, 501, 525, 501, 505, 537, 500, 515, 486, 490, 496, 508, 502, 501, 499, 507, 520, 509] payload_rows_per_dst=[10918, 11020, 10980, 10926, 11006, 11010, 11006, 10788] dup_rows_per_dst=[5298, 5456, 5478, 5404, 5504, 5308, 5556, 5414] combine_rows_per_src=[10918, 11020, 10980, 10926, 11006, 11010, 11006, 10788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626081792 dispatch_meta_bytes=458344 combine_payload_bytes=626081792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626540136 ingress_bytes=701579264 link_bytes=701579264 token_bytes=1252621928 weight_movement_bytes=75497472 total_nvlink_bytes=1328119400 weight_to_token_ratio=0.060272 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897012615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87694 payload_rows_remote=76752 dup_rows=43378 dup_rows_remote=37666 dedup_ratio=0.330948 router_tokens_per_expert=[465, 499, 506, 476, 454, 536, 497, 482, 455, 534, 528, 514, 517, 560, 518, 537, 509, 531, 515, 482, 491, 558, 519, 515, 510, 500, 508, 502, 505, 496, 492, 517, 501, 510, 544, 516, 533, 482, 494, 471, 507, 508, 556, 503, 575, 502, 523, 544, 511, 481, 502, 467, 486, 503, 494, 510, 531, 526, 510, 522, 499, 507, 535, 547, 513, 514, 512, 483, 493, 485, 535, 488, 549, 537, 499, 516, 495, 509, 515, 501, 549, 490, 504, 503, 463, 520, 539, 524, 498, 497, 550, 482, 516, 544, 497, 480, 498, 537, 497, 519, 511, 500, 533, 531, 495, 519, 537, 544, 515, 470, 515, 500, 530, 550, 510, 509, 474, 509, 542, 536, 561, 530, 510, 496, 544, 545, 511, 530] payload_rows_per_dst=[10942, 10732, 11048, 10876, 11018, 10968, 11020, 11090] dup_rows_per_dst=[5712, 5076, 5494, 5386, 5472, 5344, 5422, 5472] combine_rows_per_src=[10942, 10732, 11048, 10876, 11018, 10968, 11020, 11090] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628113408 dispatch_meta_bytes=458456 combine_payload_bytes=628113408 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628571864 ingress_bytes=779108352 link_bytes=779108352 token_bytes=1256685272 weight_movement_bytes=150994944 total_nvlink_bytes=1407680216 weight_to_token_ratio=0.120153 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.880190581e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87352 payload_rows_remote=76400 dup_rows=43720 dup_rows_remote=38040 dedup_ratio=0.333557 router_tokens_per_expert=[508, 494, 523, 515, 508, 524, 552, 512, 510, 508, 473, 500, 516, 500, 499, 517, 538, 511, 536, 501, 523, 502, 490, 530, 481, 496, 538, 533, 467, 505, 510, 524, 538, 484, 519, 514, 517, 525, 484, 517, 536, 536, 523, 513, 487, 508, 506, 527, 533, 509, 517, 530, 529, 549, 527, 515, 493, 456, 535, 534, 498, 530, 534, 527, 508, 477, 514, 502, 540, 549, 513, 535, 499, 491, 512, 518, 506, 496, 514, 477, 514, 523, 495, 513, 506, 497, 530, 503, 528, 567, 521, 516, 502, 498, 536, 530, 485, 496, 503, 471, 499, 512, 499, 522, 506, 497, 478, 503, 521, 487, 515, 518, 524, 466, 561, 494, 505, 502, 484, 523, 531, 495, 534, 542, 496, 512, 529, 502] payload_rows_per_dst=[10820, 10982, 10928, 10952, 10944, 10988, 10802, 10936] dup_rows_per_dst=[5498, 5388, 5540, 5680, 5358, 5570, 5222, 5464] combine_rows_per_src=[10820, 10982, 10928, 10952, 10944, 10988, 10802, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87362 payload_rows_remote=76512 dup_rows=43710 dup_rows_remote=38258 dedup_ratio=0.333481 router_tokens_per_expert=[543, 572, 534, 541, 516, 514, 527, 513, 512, 526, 487, 512, 530, 507, 512, 525, 538, 507, 531, 500, 514, 540, 525, 482, 529, 526, 520, 516, 513, 525, 509, 461, 503, 555, 503, 513, 512, 482, 507, 547, 502, 509, 521, 541, 499, 522, 498, 514, 515, 475, 524, 536, 499, 523, 491, 543, 499, 506, 488, 524, 477, 525, 511, 488, 529, 533, 493, 501, 543, 549, 500, 548, 504, 483, 480, 505, 511, 506, 491, 475, 481, 455, 503, 553, 514, 512, 507, 487, 511, 517, 481, 513, 482, 485, 496, 517, 534, 538, 488, 536, 512, 544, 523, 499, 494, 509, 501, 482, 486, 506, 551, 516, 473, 488, 529, 471, 496, 526, 522, 528, 498, 514, 519, 517, 520, 530, 516, 546] payload_rows_per_dst=[11042, 10898, 10988, 10930, 10850, 10788, 10972, 10894] dup_rows_per_dst=[5700, 5574, 5468, 5318, 5452, 5240, 5466, 5492] combine_rows_per_src=[11042, 10898, 10988, 10930, 10850, 10788, 10972, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87570 payload_rows_remote=76578 dup_rows=43502 dup_rows_remote=37970 dedup_ratio=0.331894 router_tokens_per_expert=[506, 519, 527, 508, 489, 517, 506, 564, 527, 479, 510, 524, 499, 511, 498, 504, 491, 534, 539, 520, 519, 497, 509, 483, 511, 560, 509, 519, 526, 498, 494, 553, 521, 499, 546, 513, 495, 511, 519, 530, 525, 535, 493, 515, 487, 529, 517, 462, 507, 518, 482, 517, 536, 514, 502, 521, 519, 493, 491, 522, 549, 518, 531, 530, 503, 536, 491, 520, 503, 535, 524, 509, 562, 487, 502, 524, 504, 491, 507, 552, 505, 508, 537, 503, 512, 516, 491, 487, 528, 473, 505, 505, 571, 529, 498, 516, 526, 485, 539, 505, 487, 466, 519, 525, 488, 520, 468, 523, 549, 458, 502, 494, 519, 499, 520, 500, 500, 526, 495, 529, 537, 501, 499, 542, 519, 468, 494, 503] payload_rows_per_dst=[10968, 10992, 10902, 10866, 11024, 10958, 10914, 10946] dup_rows_per_dst=[5408, 5532, 5492, 5634, 5476, 5410, 5194, 5356] combine_rows_per_src=[10968, 10992, 10902, 10866, 11024, 10958, 10914, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87296 payload_rows_remote=76456 dup_rows=43776 dup_rows_remote=38362 dedup_ratio=0.333984 router_tokens_per_expert=[493, 482, 495, 494, 531, 514, 546, 534, 495, 466, 506, 491, 516, 546, 523, 525, 538, 498, 510, 529, 542, 498, 514, 516, 507, 511, 496, 524, 501, 517, 551, 512, 528, 530, 526, 508, 519, 521, 497, 494, 471, 510, 531, 502, 514, 523, 500, 521, 486, 490, 497, 483, 480, 528, 523, 564, 501, 519, 553, 507, 474, 563, 507, 526, 491, 509, 515, 574, 495, 505, 499, 541, 509, 494, 516, 532, 520, 514, 548, 537, 504, 507, 510, 493, 489, 498, 491, 515, 505, 533, 500, 537, 497, 505, 507, 521, 502, 498, 503, 515, 545, 520, 512, 519, 518, 504, 491, 497, 490, 523, 493, 497, 507, 554, 494, 495, 531, 514, 505, 482, 534, 491, 513, 495, 520, 486, 514, 546] payload_rows_per_dst=[10848, 10978, 10990, 10840, 11082, 10906, 10840, 10812] dup_rows_per_dst=[5466, 5550, 5400, 5562, 5516, 5318, 5414, 5550] combine_rows_per_src=[10848, 10978, 10990, 10840, 11082, 10906, 10840, 10812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628752384 dispatch_meta_bytes=457672 combine_payload_bytes=628752384 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629210056 ingress_bytes=704249856 link_bytes=704249856 token_bytes=1257962440 weight_movement_bytes=75497472 total_nvlink_bytes=1333459912 weight_to_token_ratio=0.060016 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930879481e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625868800 dispatch_meta_bytes=457760 combine_payload_bytes=625868800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626326560 ingress_bytes=701366272 link_bytes=701366272 token_bytes=1252195360 weight_movement_bytes=75497472 total_nvlink_bytes=1327692832 weight_to_token_ratio=0.060292 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894311577e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626786304 dispatch_meta_bytes=459080 combine_payload_bytes=626786304 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627245384 ingress_bytes=853278720 link_bytes=853278720 token_bytes=1254031688 weight_movement_bytes=226492416 total_nvlink_bytes=1480524104 weight_to_token_ratio=0.180611 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082077525e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=458192 combine_payload_bytes=627326976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627785168 ingress_bytes=702824448 link_bytes=702824448 token_bytes=1255112144 weight_movement_bytes=75497472 total_nvlink_bytes=1330609616 weight_to_token_ratio=0.060152 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912803301e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87208 payload_rows_remote=76228 dup_rows=43864 dup_rows_remote=38354 dedup_ratio=0.334656 router_tokens_per_expert=[515, 510, 493, 477, 544, 507, 558, 499, 510, 553, 521, 481, 517, 509, 534, 521, 524, 509, 493, 533, 530, 497, 497, 523, 506, 465, 485, 501, 497, 521, 515, 499, 506, 523, 542, 533, 497, 550, 575, 516, 517, 524, 499, 528, 488, 531, 525, 509, 518, 480, 534, 509, 488, 494, 539, 524, 523, 461, 505, 489, 497, 529, 518, 504, 521, 520, 519, 530, 547, 532, 491, 516, 524, 518, 534, 563, 480, 487, 501, 497, 517, 535, 512, 555, 514, 541, 517, 511, 520, 507, 491, 510, 491, 514, 519, 491, 501, 506, 495, 492, 452, 541, 546, 500, 527, 489, 501, 495, 494, 499, 468, 516, 542, 518, 519, 518, 496, 511, 517, 487, 512, 491, 512, 527, 479, 522, 494, 525] payload_rows_per_dst=[10890, 10794, 11090, 10880, 11028, 10980, 10712, 10834] dup_rows_per_dst=[5608, 5396, 5636, 5344, 5532, 5510, 5332, 5506] combine_rows_per_src=[10890, 10794, 11090, 10880, 11028, 10980, 10712, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626327552 dispatch_meta_bytes=459272 combine_payload_bytes=626327552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626786824 ingress_bytes=701825024 link_bytes=701825024 token_bytes=1253114376 weight_movement_bytes=75497472 total_nvlink_bytes=1328611848 weight_to_token_ratio=0.060248 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900129198e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76450 dup_rows=43554 dup_rows_remote=37994 dedup_ratio=0.332291 router_tokens_per_expert=[518, 507, 515, 523, 554, 516, 508, 497, 490, 510, 525, 530, 496, 533, 506, 505, 514, 504, 535, 515, 506, 493, 505, 519, 523, 525, 478, 521, 500, 473, 506, 495, 513, 525, 511, 549, 552, 508, 523, 490, 491, 506, 517, 503, 490, 514, 493, 514, 486, 495, 534, 516, 527, 456, 515, 509, 519, 536, 518, 548, 527, 486, 520, 513, 504, 502, 496, 502, 509, 489, 524, 526, 467, 500, 513, 512, 485, 521, 540, 522, 516, 503, 470, 515, 463, 514, 531, 517, 478, 497, 515, 542, 459, 498, 529, 537, 504, 563, 538, 542, 496, 497, 510, 529, 531, 521, 499, 547, 555, 541, 548, 559, 470, 487, 481, 537, 548, 490, 506, 524, 497, 507, 513, 530, 488, 485, 546, 502] payload_rows_per_dst=[10842, 10850, 10640, 10952, 10750, 11064, 11352, 11068] dup_rows_per_dst=[5218, 5374, 5140, 5458, 5080, 5516, 6208, 5560] combine_rows_per_src=[10842, 10850, 10640, 10952, 10750, 11064, 11352, 11068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87536 payload_rows_remote=76760 dup_rows=43536 dup_rows_remote=37956 dedup_ratio=0.332153 router_tokens_per_expert=[527, 507, 575, 525, 539, 506, 486, 499, 481, 497, 519, 555, 507, 513, 506, 523, 528, 539, 520, 516, 506, 464, 492, 539, 497, 518, 525, 532, 503, 526, 518, 506, 482, 510, 524, 530, 492, 522, 492, 500, 504, 514, 544, 497, 522, 493, 529, 523, 484, 508, 540, 486, 507, 522, 513, 480, 504, 514, 544, 516, 481, 546, 479, 508, 521, 476, 533, 459, 529, 501, 525, 484, 511, 536, 539, 533, 518, 542, 582, 514, 470, 531, 465, 504, 484, 517, 541, 529, 505, 510, 520, 504, 551, 506, 484, 498, 471, 505, 496, 484, 531, 561, 515, 523, 483, 471, 515, 517, 475, 505, 461, 530, 497, 547, 519, 521, 509, 494, 528, 515, 518, 569, 466, 504, 518, 543, 527, 492] payload_rows_per_dst=[10964, 10946, 10776, 10930, 11046, 10934, 10876, 11064] dup_rows_per_dst=[5566, 5512, 5580, 5334, 5560, 5304, 5210, 5470] combine_rows_per_src=[10964, 10946, 10776, 10930, 11046, 10934, 10876, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=624459776 dispatch_meta_bytes=458328 combine_payload_bytes=624459776 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624918104 ingress_bytes=699957248 link_bytes=699957248 token_bytes=1249377880 weight_movement_bytes=75497472 total_nvlink_bytes=1324875352 weight_to_token_ratio=0.060428 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.876443169e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626278400 dispatch_meta_bytes=457776 combine_payload_bytes=626278400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626736176 ingress_bytes=701775872 link_bytes=701775872 token_bytes=1253014576 weight_movement_bytes=75497472 total_nvlink_bytes=1328512048 weight_to_token_ratio=0.060253 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899505881e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87478 payload_rows_remote=76628 dup_rows=43594 dup_rows_remote=38210 dedup_ratio=0.332596 router_tokens_per_expert=[533, 490, 507, 522, 528, 495, 511, 483, 516, 508, 497, 490, 497, 498, 524, 503, 514, 552, 484, 518, 532, 534, 547, 517, 518, 490, 504, 475, 506, 474, 502, 443, 506, 530, 526, 524, 527, 574, 528, 506, 530, 515, 507, 516, 498, 504, 519, 499, 471, 543, 507, 486, 504, 489, 532, 506, 502, 517, 504, 522, 496, 510, 546, 519, 527, 492, 502, 498, 542, 541, 504, 507, 512, 489, 499, 523, 510, 528, 527, 528, 484, 503, 522, 535, 502, 530, 507, 486, 553, 531, 543, 530, 543, 511, 516, 497, 550, 504, 532, 486, 492, 523, 532, 537, 511, 532, 493, 540, 471, 533, 531, 512, 503, 534, 528, 500, 468, 512, 518, 454, 499, 492, 496, 509, 503, 515, 516, 513] payload_rows_per_dst=[10850, 10870, 11062, 10978, 10934, 11012, 11012, 10760] dup_rows_per_dst=[5384, 5436, 5440, 5414, 5440, 5574, 5546, 5360] combine_rows_per_src=[10850, 10870, 11062, 10978, 10934, 11012, 11012, 10760] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87340 payload_rows_remote=76328 dup_rows=43732 dup_rows_remote=38112 dedup_ratio=0.333649 router_tokens_per_expert=[524, 498, 534, 555, 522, 494, 504, 516, 490, 494, 559, 484, 536, 518, 486, 453, 509, 526, 535, 530, 480, 531, 517, 507, 492, 489, 500, 508, 484, 508, 506, 498, 472, 511, 525, 504, 496, 502, 487, 536, 494, 535, 528, 539, 499, 482, 488, 541, 521, 525, 536, 485, 509, 514, 522, 502, 577, 507, 517, 523, 527, 542, 495, 514, 534, 484, 494, 510, 501, 501, 515, 483, 511, 521, 519, 511, 560, 503, 518, 474, 500, 486, 492, 496, 566, 535, 511, 477, 543, 535, 532, 493, 492, 504, 511, 481, 525, 543, 527, 495, 507, 558, 534, 531, 501, 535, 540, 527, 486, 486, 492, 499, 514, 532, 514, 454, 494, 530, 521, 493, 554, 539, 498, 506, 488, 559, 517, 502] payload_rows_per_dst=[10916, 10756, 10854, 11012, 10884, 10940, 11036, 10942] dup_rows_per_dst=[5418, 5484, 5424, 5620, 5394, 5368, 5536, 5488] combine_rows_per_src=[10916, 10756, 10854, 11012, 10884, 10940, 11036, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87716 payload_rows_remote=76574 dup_rows=43356 dup_rows_remote=37824 dedup_ratio=0.330780 router_tokens_per_expert=[483, 534, 509, 472, 514, 478, 535, 516, 485, 520, 483, 492, 489, 538, 516, 542, 524, 527, 514, 506, 501, 479, 485, 525, 531, 494, 519, 524, 500, 538, 521, 484, 522, 520, 497, 553, 507, 519, 542, 516, 533, 486, 560, 521, 503, 517, 512, 490, 519, 517, 473, 517, 550, 503, 539, 524, 530, 481, 525, 543, 532, 511, 511, 556, 527, 476, 508, 540, 544, 522, 573, 479, 507, 515, 513, 524, 550, 509, 564, 486, 499, 510, 517, 486, 502, 474, 487, 514, 471, 519, 505, 520, 495, 502, 520, 485, 523, 504, 497, 527, 528, 483, 502, 508, 527, 494, 506, 538, 478, 520, 487, 501, 505, 540, 513, 471, 497, 501, 510, 485, 490, 548, 528, 522, 491, 520, 517, 525] payload_rows_per_dst=[10894, 11004, 11026, 11072, 11142, 10794, 10930, 10854] dup_rows_per_dst=[5318, 5340, 5570, 5590, 5532, 5218, 5316, 5472] combine_rows_per_src=[10894, 11004, 11026, 11072, 11142, 10794, 10930, 10854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628817920 dispatch_meta_bytes=458864 combine_payload_bytes=628817920 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=629276784 ingress_bytes=930807808 link_bytes=930807808 token_bytes=1258094704 weight_movement_bytes=301989888 total_nvlink_bytes=1560084592 weight_to_token_ratio=0.240037 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.180395322e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87448 payload_rows_remote=76492 dup_rows=43624 dup_rows_remote=38164 dedup_ratio=0.332825 router_tokens_per_expert=[502, 553, 491, 488, 524, 543, 521, 512, 553, 519, 476, 511, 540, 470, 547, 510, 477, 516, 520, 514, 484, 525, 519, 516, 532, 477, 495, 550, 490, 524, 526, 543, 504, 541, 471, 545, 512, 523, 498, 532, 514, 514, 513, 525, 546, 488, 518, 491, 505, 491, 514, 524, 501, 496, 497, 515, 524, 486, 504, 512, 515, 553, 502, 506, 503, 517, 524, 516, 520, 509, 508, 554, 525, 504, 472, 494, 528, 499, 491, 483, 531, 494, 511, 530, 501, 508, 526, 505, 501, 518, 521, 504, 554, 501, 488, 518, 540, 482, 528, 517, 513, 505, 521, 476, 553, 514, 542, 504, 524, 535, 526, 524, 513, 522, 508, 491, 524, 468, 466, 497, 473, 484, 519, 512, 490, 532, 476, 551] payload_rows_per_dst=[10962, 10956, 11022, 10900, 10894, 10978, 10986, 10750] dup_rows_per_dst=[5558, 5460, 5448, 5390, 5400, 5444, 5622, 5302] combine_rows_per_src=[10962, 10956, 11022, 10900, 10894, 10978, 10986, 10750] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87568 payload_rows_remote=76566 dup_rows=43504 dup_rows_remote=38078 dedup_ratio=0.331909 router_tokens_per_expert=[518, 481, 500, 483, 527, 500, 512, 512, 529, 534, 503, 496, 534, 506, 535, 524, 506, 529, 489, 494, 537, 540, 500, 489, 519, 539, 523, 530, 502, 527, 492, 509, 508, 507, 514, 498, 524, 494, 474, 520, 530, 542, 473, 476, 517, 539, 522, 494, 519, 542, 513, 505, 557, 507, 529, 513, 508, 535, 456, 515, 517, 512, 497, 522, 489, 529, 517, 518, 521, 510, 527, 528, 513, 573, 538, 451, 539, 530, 476, 519, 462, 532, 501, 487, 518, 540, 516, 508, 525, 543, 491, 487, 513, 488, 494, 478, 527, 481, 502, 526, 509, 506, 521, 528, 497, 506, 516, 503, 548, 546, 500, 498, 455, 556, 490, 481, 484, 519, 468, 526, 514, 511, 522, 515, 565, 509, 533, 515] payload_rows_per_dst=[10970, 10974, 10848, 10980, 10970, 10868, 11002, 10956] dup_rows_per_dst=[5418, 5476, 5416, 5514, 5520, 5298, 5426, 5436] combine_rows_per_src=[10970, 10974, 10848, 10980, 10970, 10868, 11002, 10956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627736576 dispatch_meta_bytes=459352 combine_payload_bytes=627736576 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628195928 ingress_bytes=778731520 link_bytes=778731520 token_bytes=1255932504 weight_movement_bytes=150994944 total_nvlink_bytes=1406927448 weight_to_token_ratio=0.120225 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.875411821e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625278976 dispatch_meta_bytes=457760 combine_payload_bytes=625278976 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=625736736 ingress_bytes=927268864 link_bytes=927268864 token_bytes=1251015712 weight_movement_bytes=301989888 total_nvlink_bytes=1553005600 weight_to_token_ratio=0.241396 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.175907443e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627294208 dispatch_meta_bytes=457592 combine_payload_bytes=627294208 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627751800 ingress_bytes=778289152 link_bytes=778289152 token_bytes=1255046008 weight_movement_bytes=150994944 total_nvlink_bytes=1406040952 weight_to_token_ratio=0.120310 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.869801972e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626622464 dispatch_meta_bytes=458624 combine_payload_bytes=626622464 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627081088 ingress_bytes=777617408 link_bytes=777617408 token_bytes=1253703552 weight_movement_bytes=150994944 total_nvlink_bytes=1404698496 weight_to_token_ratio=0.120439 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861283313e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87740 payload_rows_remote=76700 dup_rows=43332 dup_rows_remote=37964 dedup_ratio=0.330597 router_tokens_per_expert=[515, 534, 527, 473, 514, 514, 504, 483, 505, 527, 505, 500, 528, 493, 473, 507, 553, 468, 517, 471, 517, 526, 504, 538, 493, 522, 489, 507, 532, 514, 492, 546, 538, 495, 529, 504, 500, 487, 516, 537, 526, 510, 506, 479, 483, 511, 505, 560, 563, 506, 500, 477, 536, 504, 529, 504, 527, 505, 523, 505, 514, 520, 498, 492, 536, 538, 522, 487, 489, 526, 501, 486, 502, 564, 533, 526, 494, 496, 567, 524, 533, 522, 503, 532, 539, 556, 483, 528, 518, 476, 501, 495, 484, 495, 557, 482, 518, 534, 528, 476, 515, 486, 510, 493, 483, 498, 547, 476, 539, 521, 541, 542, 563, 490, 478, 485, 528, 510, 549, 467, 523, 541, 491, 495, 515, 478, 509, 532] payload_rows_per_dst=[10914, 11028, 10884, 10926, 11042, 11040, 11012, 10894] dup_rows_per_dst=[5290, 5350, 5488, 5480, 5540, 5368, 5402, 5414] combine_rows_per_src=[10914, 11028, 10884, 10926, 11042, 11040, 11012, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76112 dup_rows=43666 dup_rows_remote=37642 dedup_ratio=0.333145 router_tokens_per_expert=[470, 523, 522, 521, 486, 501, 511, 519, 533, 497, 493, 540, 525, 498, 529, 548, 511, 550, 523, 495, 543, 557, 496, 492, 499, 571, 495, 499, 533, 527, 476, 517, 524, 530, 469, 518, 551, 496, 513, 524, 518, 537, 550, 491, 526, 483, 513, 526, 551, 522, 501, 482, 535, 499, 526, 493, 528, 480, 492, 537, 513, 489, 512, 506, 530, 493, 521, 505, 506, 550, 495, 498, 522, 530, 527, 529, 510, 533, 536, 481, 506, 524, 505, 476, 465, 517, 484, 528, 524, 501, 496, 535, 483, 517, 563, 528, 565, 516, 487, 469, 545, 494, 509, 484, 490, 501, 486, 461, 508, 508, 492, 519, 510, 532, 541, 479, 472, 518, 505, 526, 498, 489, 510, 513, 549, 507, 498, 502] payload_rows_per_dst=[10934, 10960, 10642, 10860, 10660, 11232, 10824, 11294] dup_rows_per_dst=[5498, 5528, 4940, 5462, 4926, 6028, 5260, 6024] combine_rows_per_src=[10934, 10960, 10642, 10860, 10660, 11232, 10824, 11294] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627228672 dispatch_meta_bytes=458576 combine_payload_bytes=627228672 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627687248 ingress_bytes=702726144 link_bytes=702726144 token_bytes=1254915920 weight_movement_bytes=75497472 total_nvlink_bytes=1330413392 weight_to_token_ratio=0.060161 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911556668e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15047752408 prefetch_bytes=905969664 reduce_bytes=1132462080 weight_movement_bytes=2038431744 egress_bytes=7526628056 ingress_bytes=9559556096 link_bytes=9559556096 total_nvlink_bytes=17086184152 weight_to_token_ratio=0.135464 prefetch_share_of_ingress=0.094771 dedup_ratio=0.332789 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=15 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041973 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=1.010238562e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15067218096 prefetch_bytes=754974720 reduce_bytes=1132462080 weight_movement_bytes=1887436800 egress_bytes=7536361648 ingress_bytes=9418293248 link_bytes=9418293248 total_nvlink_bytes=16954654896 weight_to_token_ratio=0.125268 prefetch_share_of_ingress=0.080160 dedup_ratio=0.332124 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=15 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.953101303e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15037263568 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=7521382096 ingress_bytes=9176825856 link_bytes=9176825856 total_nvlink_bytes=16698207952 weight_to_token_ratio=0.110455 prefetch_share_of_ingress=0.090497 dedup_ratio=0.332874 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043811 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.697922434e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76384 dup_rows=43666 dup_rows_remote=37932 dedup_ratio=0.333145 router_tokens_per_expert=[520, 491, 502, 524, 511, 540, 519, 513, 500, 517, 500, 496, 503, 522, 519, 510, 538, 518, 529, 523, 497, 517, 492, 509, 521, 522, 519, 507, 475, 520, 503, 493, 540, 531, 515, 534, 517, 527, 530, 514, 514, 525, 524, 548, 497, 521, 518, 523, 518, 551, 520, 504, 504, 504, 509, 499, 503, 503, 559, 503, 526, 472, 521, 495, 537, 510, 524, 510, 536, 521, 525, 518, 497, 508, 536, 511, 534, 523, 511, 515, 504, 484, 516, 474, 530, 531, 501, 528, 516, 498, 533, 481, 515, 531, 526, 543, 513, 473, 493, 535, 517, 497, 520, 508, 538, 477, 481, 537, 513, 494, 521, 507, 444, 477, 488, 508, 521, 529, 522, 499, 501, 478, 523, 490, 469, 516, 505, 476] payload_rows_per_dst=[10896, 10986, 11022, 10898, 11108, 10966, 10896, 10634] dup_rows_per_dst=[5478, 5380, 5734, 5484, 5524, 5456, 5352, 5258] combine_rows_per_src=[10896, 10986, 11022, 10898, 11108, 10966, 10896, 10634] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628326400 dispatch_meta_bytes=458656 combine_payload_bytes=628326400 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628785056 ingress_bytes=703823872 link_bytes=703823872 token_bytes=1257111456 weight_movement_bytes=75497472 total_nvlink_bytes=1332608928 weight_to_token_ratio=0.060056 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925477404e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=623509504 dispatch_meta_bytes=455016 combine_payload_bytes=623509504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623964520 ingress_bytes=699006976 link_bytes=699006976 token_bytes=1247474024 weight_movement_bytes=75497472 total_nvlink_bytes=1322971496 weight_to_token_ratio=0.060520 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.864392382e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15051127920 prefetch_bytes=830472192 reduce_bytes=226492416 weight_movement_bytes=1056964608 egress_bytes=7528316016 ingress_bytes=8579776512 link_bytes=8579776512 total_nvlink_bytes=16108092528 weight_to_token_ratio=0.070225 prefetch_share_of_ingress=0.096794 dedup_ratio=0.332561 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=3 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.066970260e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87430 payload_rows_remote=76408 dup_rows=43642 dup_rows_remote=38056 dedup_ratio=0.332962 router_tokens_per_expert=[541, 482, 466, 495, 502, 528, 502, 532, 473, 557, 544, 556, 504, 491, 529, 491, 490, 505, 518, 496, 499, 479, 555, 532, 515, 542, 501, 519, 541, 501, 501, 501, 561, 492, 500, 514, 535, 496, 530, 540, 514, 507, 479, 493, 513, 524, 514, 500, 509, 515, 500, 498, 556, 536, 505, 460, 541, 489, 488, 529, 514, 535, 535, 508, 513, 510, 493, 526, 513, 515, 511, 525, 531, 498, 512, 522, 521, 508, 534, 536, 505, 516, 511, 485, 521, 513, 530, 480, 503, 533, 510, 501, 511, 488, 460, 470, 521, 489, 546, 511, 504, 526, 533, 521, 517, 479, 520, 529, 512, 494, 537, 497, 527, 490, 507, 554, 525, 496, 512, 486, 497, 494, 504, 517, 507, 517, 510, 534] payload_rows_per_dst=[11022, 10934, 11094, 10734, 11038, 10764, 10920, 10924] dup_rows_per_dst=[5586, 5456, 5660, 5150, 5498, 5310, 5552, 5430] combine_rows_per_src=[11022, 10934, 11094, 10734, 11038, 10764, 10920, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625737728 dispatch_meta_bytes=457264 combine_payload_bytes=625737728 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626194992 ingress_bytes=776732672 link_bytes=776732672 token_bytes=1251932720 weight_movement_bytes=150994944 total_nvlink_bytes=1402927664 weight_to_token_ratio=0.120609 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850063615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15041166464 prefetch_bytes=830472192 reduce_bytes=301989888 weight_movement_bytes=1132462080 egress_bytes=7523335296 ingress_bytes=8650293248 link_bytes=8650753168 total_nvlink_bytes=16173628544 weight_to_token_ratio=0.075291 prefetch_share_of_ingress=0.096005 dedup_ratio=0.332976 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=4 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.141977252e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15033981160 prefetch_bytes=754974720 reduce_bytes=528482304 weight_movement_bytes=1283457024 egress_bytes=7519738088 ingress_bytes=8797700096 link_bytes=8797700096 total_nvlink_bytes=16317438184 weight_to_token_ratio=0.085370 prefetch_share_of_ingress=0.085815 dedup_ratio=0.332779 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=7 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.045190 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.297268410e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=7 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=457856 combine_payload_bytes=625934336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626392192 ingress_bytes=701431808 link_bytes=701431808 token_bytes=1252326528 weight_movement_bytes=75497472 total_nvlink_bytes=1327824000 weight_to_token_ratio=0.060286 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895142665e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15060134384 prefetch_bytes=830472192 reduce_bytes=754974720 weight_movement_bytes=1585446912 egress_bytes=7532816880 ingress_bytes=9112764416 link_bytes=9112764416 total_nvlink_bytes=16645581296 weight_to_token_ratio=0.105274 prefetch_share_of_ingress=0.091133 dedup_ratio=0.331991 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=10 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042586 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.630223332e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=7 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15036770520 prefetch_bytes=452984832 reduce_bytes=1283457024 weight_movement_bytes=1736441856 egress_bytes=7521134808 ingress_bytes=9252077568 link_bytes=9252077568 total_nvlink_bytes=16773212376 weight_to_token_ratio=0.115480 prefetch_share_of_ingress=0.048960 dedup_ratio=0.332946 prefetch_slots=12 unused_prefetch_slots=372 slot_utilization=0.031250 reduce_slots=17 remote_demand=6 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=26670 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.777447237e-05 prefetch_bytes_per_gemm_flop=4.787071078e-06 anomalies=none + [2026-08-29 23:36:24] iteration 8/ 12 | consumed samples: 64 | elapsed time per iteration (ms): 358.8 | throughput per GPU (TFLOP/s/GPU): 504.0 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.799053E+00 | loss scale: 1.0 | grad norm: 5.601 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87264 payload_rows_remote=76368 dup_rows=43808 dup_rows_remote=38316 dedup_ratio=0.334229 router_tokens_per_expert=[527, 542, 501, 545, 515, 504, 522, 517, 492, 500, 506, 509, 518, 533, 556, 511, 551, 527, 504, 453, 509, 508, 511, 555, 535, 471, 531, 486, 520, 518, 507, 508, 494, 530, 514, 483, 523, 514, 508, 523, 514, 495, 560, 487, 505, 527, 545, 498, 484, 503, 522, 488, 510, 503, 516, 492, 501, 550, 523, 489, 473, 505, 537, 534, 474, 504, 513, 512, 529, 535, 480, 490, 528, 490, 516, 529, 501, 507, 497, 511, 487, 514, 497, 550, 525, 467, 516, 535, 499, 518, 520, 525, 536, 531, 467, 526, 496, 495, 514, 520, 507, 516, 504, 503, 560, 531, 497, 522, 493, 513, 543, 499, 483, 497, 543, 485, 509, 524, 503, 504, 545, 501, 478, 509, 512, 536, 494, 529] payload_rows_per_dst=[11034, 10896, 10910, 10820, 10872, 10956, 10920, 10856] dup_rows_per_dst=[5562, 5492, 5530, 5440, 5360, 5470, 5506, 5448] combine_rows_per_src=[11034, 10896, 10910, 10820, 10872, 10956, 10920, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87496 payload_rows_remote=76432 dup_rows=43576 dup_rows_remote=37984 dedup_ratio=0.332458 router_tokens_per_expert=[558, 546, 507, 491, 563, 489, 497, 495, 517, 508, 530, 492, 477, 533, 512, 502, 500, 523, 494, 527, 472, 505, 510, 521, 538, 515, 479, 488, 534, 509, 521, 518, 514, 451, 475, 513, 552, 537, 527, 496, 549, 512, 527, 490, 490, 470, 530, 488, 504, 532, 520, 511, 539, 539, 487, 464, 541, 525, 503, 538, 518, 503, 501, 491, 484, 526, 524, 521, 504, 478, 505, 519, 545, 512, 515, 518, 509, 511, 529, 513, 549, 493, 521, 534, 523, 529, 519, 521, 532, 528, 530, 516, 513, 544, 468, 508, 493, 503, 510, 510, 520, 511, 537, 553, 514, 502, 510, 504, 505, 506, 495, 523, 494, 495, 523, 458, 527, 527, 512, 524, 518, 500, 491, 474, 533, 474, 514, 527] payload_rows_per_dst=[10920, 10872, 10864, 10962, 10966, 11064, 10890, 10958] dup_rows_per_dst=[5514, 5436, 5378, 5470, 5460, 5592, 5502, 5224] combine_rows_per_src=[10920, 10872, 10864, 10962, 10966, 11064, 10890, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87532 payload_rows_remote=76468 dup_rows=43540 dup_rows_remote=38112 dedup_ratio=0.332184 router_tokens_per_expert=[510, 496, 502, 545, 487, 513, 433, 513, 500, 475, 531, 524, 504, 534, 529, 495, 481, 504, 536, 523, 506, 532, 553, 498, 500, 504, 506, 517, 528, 502, 519, 483, 500, 530, 533, 525, 512, 479, 533, 517, 538, 533, 495, 499, 515, 495, 524, 518, 494, 514, 504, 518, 555, 528, 551, 524, 492, 490, 455, 552, 519, 523, 536, 566, 552, 555, 496, 515, 521, 488, 490, 551, 488, 485, 480, 515, 512, 495, 500, 537, 500, 482, 513, 520, 537, 468, 518, 481, 499, 507, 518, 523, 529, 539, 482, 527, 473, 492, 524, 514, 535, 522, 545, 538, 486, 523, 476, 492, 527, 491, 530, 544, 519, 531, 492, 497, 494, 488, 512, 517, 467, 553, 485, 522, 509, 527, 504, 534] payload_rows_per_dst=[10872, 10896, 11064, 11004, 10912, 10916, 10944, 10924] dup_rows_per_dst=[5310, 5488, 5428, 5638, 5448, 5370, 5480, 5378] combine_rows_per_src=[10872, 10896, 11064, 11004, 10912, 10916, 10944, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87462 payload_rows_remote=76542 dup_rows=43610 dup_rows_remote=38244 dedup_ratio=0.332718 router_tokens_per_expert=[465, 541, 530, 517, 542, 521, 503, 512, 504, 515, 512, 513, 498, 509, 488, 500, 523, 482, 523, 500, 514, 512, 498, 538, 487, 523, 520, 459, 541, 523, 519, 508, 490, 536, 526, 510, 537, 504, 526, 576, 515, 506, 476, 508, 501, 531, 513, 529, 500, 494, 510, 528, 533, 550, 521, 495, 519, 511, 525, 508, 487, 534, 487, 557, 520, 533, 503, 495, 549, 501, 516, 508, 475, 485, 494, 529, 485, 450, 496, 514, 501, 515, 536, 503, 502, 491, 491, 516, 484, 529, 511, 544, 535, 541, 469, 538, 465, 473, 538, 502, 549, 521, 504, 515, 507, 510, 530, 505, 537, 479, 501, 507, 551, 496, 521, 510, 524, 528, 533, 517, 520, 541, 513, 500, 494, 511, 512, 480] payload_rows_per_dst=[10828, 10902, 11038, 11072, 10776, 10962, 10920, 10964] dup_rows_per_dst=[5512, 5438, 5530, 5446, 5330, 5450, 5366, 5538] combine_rows_per_src=[10828, 10902, 11038, 11072, 10776, 10962, 10920, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87760 payload_rows_remote=76774 dup_rows=43312 dup_rows_remote=37948 dedup_ratio=0.330444 router_tokens_per_expert=[510, 526, 528, 519, 559, 500, 500, 484, 529, 514, 487, 510, 511, 513, 480, 507, 532, 530, 532, 473, 516, 463, 511, 507, 528, 516, 506, 489, 523, 509, 485, 544, 485, 509, 482, 553, 557, 481, 526, 548, 514, 520, 553, 524, 496, 501, 560, 514, 490, 505, 510, 510, 546, 503, 476, 541, 516, 515, 476, 494, 533, 503, 556, 501, 516, 487, 550, 531, 526, 545, 530, 522, 510, 513, 546, 488, 489, 486, 526, 521, 528, 509, 500, 507, 497, 530, 485, 526, 511, 504, 502, 496, 476, 466, 529, 487, 496, 524, 498, 478, 489, 530, 503, 497, 497, 525, 516, 520, 536, 503, 533, 523, 542, 517, 526, 511, 503, 496, 505, 519, 510, 497, 476, 519, 562, 513, 507, 487] payload_rows_per_dst=[10940, 10938, 11060, 10986, 11016, 10898, 10904, 11018] dup_rows_per_dst=[5414, 5390, 5586, 5364, 5556, 5208, 5432, 5362] combine_rows_per_src=[10940, 10938, 11060, 10986, 11016, 10898, 10904, 11018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87758 payload_rows_remote=76676 dup_rows=43314 dup_rows_remote=37724 dedup_ratio=0.330460 router_tokens_per_expert=[524, 520, 511, 512, 540, 509, 505, 522, 511, 540, 513, 521, 478, 559, 500, 525, 507, 525, 521, 475, 533, 503, 474, 506, 512, 495, 522, 484, 488, 510, 501, 483, 489, 501, 513, 509, 524, 523, 509, 530, 499, 462, 524, 508, 530, 500, 513, 540, 515, 488, 499, 505, 502, 466, 541, 505, 525, 538, 523, 480, 540, 516, 525, 495, 528, 533, 488, 563, 515, 508, 505, 505, 482, 538, 497, 568, 474, 509, 512, 513, 544, 543, 492, 465, 506, 502, 488, 528, 485, 539, 512, 514, 528, 516, 515, 529, 509, 511, 528, 494, 521, 510, 525, 498, 498, 514, 528, 532, 533, 481, 515, 501, 537, 523, 494, 505, 526, 500, 474, 487, 515, 507, 543, 524, 513, 535, 514, 531] payload_rows_per_dst=[11012, 10866, 10770, 10958, 11078, 10944, 11048, 11082] dup_rows_per_dst=[5568, 5212, 5242, 5368, 5518, 5468, 5348, 5590] combine_rows_per_src=[11012, 10866, 10770, 10958, 11078, 10944, 11048, 11082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87272 payload_rows_remote=76386 dup_rows=43800 dup_rows_remote=38436 dedup_ratio=0.334167 router_tokens_per_expert=[516, 525, 501, 499, 509, 524, 464, 491, 511, 500, 518, 476, 499, 541, 504, 509, 492, 502, 502, 507, 471, 549, 515, 573, 472, 516, 465, 497, 505, 531, 556, 486, 522, 533, 526, 528, 499, 514, 499, 505, 510, 519, 484, 513, 495, 521, 506, 477, 526, 545, 512, 493, 539, 534, 505, 466, 548, 540, 511, 538, 565, 504, 512, 549, 465, 524, 500, 487, 497, 547, 502, 507, 541, 519, 507, 531, 508, 496, 485, 509, 466, 520, 540, 510, 486, 539, 509, 494, 512, 531, 491, 530, 515, 472, 524, 529, 519, 504, 546, 538, 489, 552, 483, 498, 521, 503, 481, 525, 504, 555, 529, 507, 520, 523, 491, 530, 476, 553, 502, 528, 501, 503, 501, 561, 530, 492, 527, 487] payload_rows_per_dst=[10820, 10840, 10868, 11076, 10886, 10848, 10930, 11004] dup_rows_per_dst=[5354, 5438, 5434, 5698, 5364, 5488, 5578, 5446] combine_rows_per_src=[10820, 10840, 10868, 11076, 10886, 10848, 10930, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625606656 dispatch_meta_bytes=458736 combine_payload_bytes=625606656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626065392 ingress_bytes=701104128 link_bytes=701104128 token_bytes=1251672048 weight_movement_bytes=75497472 total_nvlink_bytes=1327169520 weight_to_token_ratio=0.060317 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890987222e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626130944 dispatch_meta_bytes=457664 combine_payload_bytes=626130944 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626588608 ingress_bytes=777125888 link_bytes=777125888 token_bytes=1252719552 weight_movement_bytes=150994944 total_nvlink_bytes=1403714496 weight_to_token_ratio=0.120534 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.855050147e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627032064 dispatch_meta_bytes=459144 combine_payload_bytes=627032064 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627491208 ingress_bytes=778027008 link_bytes=778027008 token_bytes=1254523272 weight_movement_bytes=150994944 total_nvlink_bytes=1405518216 weight_to_token_ratio=0.120360 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866477617e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=458320 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626884176 ingress_bytes=928415744 link_bytes=928415744 token_bytes=1253310032 weight_movement_bytes=301989888 total_nvlink_bytes=1555299920 weight_to_token_ratio=0.240954 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177361848e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628932608 dispatch_meta_bytes=458888 combine_payload_bytes=628932608 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629391496 ingress_bytes=779927552 link_bytes=779927552 token_bytes=1258324104 weight_movement_bytes=150994944 total_nvlink_bytes=1409319048 weight_to_token_ratio=0.119997 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.890579190e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=457600 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628587392 ingress_bytes=703627264 link_bytes=703627264 token_bytes=1256717184 weight_movement_bytes=75497472 total_nvlink_bytes=1332214656 weight_to_token_ratio=0.060075 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922984138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=459288 combine_payload_bytes=625754112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626213400 ingress_bytes=701251584 link_bytes=701251584 token_bytes=1251967512 weight_movement_bytes=75497472 total_nvlink_bytes=1327464984 weight_to_token_ratio=0.060303 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892857171e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87498 payload_rows_remote=76594 dup_rows=43574 dup_rows_remote=38066 dedup_ratio=0.332443 router_tokens_per_expert=[523, 452, 528, 510, 499, 490, 521, 524, 469, 464, 554, 526, 547, 495, 517, 477, 520, 497, 521, 525, 513, 515, 502, 473, 486, 468, 495, 510, 481, 517, 541, 494, 537, 507, 545, 510, 564, 516, 526, 504, 487, 517, 498, 513, 525, 486, 522, 532, 531, 509, 487, 499, 512, 500, 503, 511, 581, 514, 502, 511, 487, 515, 529, 522, 525, 500, 529, 517, 525, 509, 506, 505, 509, 553, 522, 492, 534, 519, 505, 515, 515, 550, 507, 533, 520, 545, 543, 530, 490, 477, 537, 501, 531, 504, 497, 523, 530, 510, 469, 480, 514, 502, 501, 474, 545, 482, 544, 511, 501, 558, 536, 515, 493, 502, 515, 534, 514, 522, 463, 512, 526, 501, 513, 485, 516, 506, 536, 502] payload_rows_per_dst=[10904, 11150, 10916, 10710, 11076, 10930, 10880, 10932] dup_rows_per_dst=[5508, 5818, 5454, 5060, 5454, 5508, 5424, 5348] combine_rows_per_src=[10904, 11150, 10916, 10710, 11076, 10930, 10880, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87508 payload_rows_remote=76546 dup_rows=43564 dup_rows_remote=37908 dedup_ratio=0.332367 router_tokens_per_expert=[508, 551, 511, 554, 491, 496, 488, 555, 522, 504, 513, 496, 538, 517, 512, 508, 514, 470, 508, 531, 550, 559, 529, 512, 525, 515, 491, 496, 526, 538, 536, 509, 522, 515, 531, 554, 487, 472, 507, 514, 514, 470, 514, 509, 524, 478, 579, 492, 515, 444, 511, 521, 529, 539, 508, 465, 529, 515, 544, 508, 504, 519, 518, 481, 502, 485, 529, 581, 504, 527, 499, 473, 487, 524, 503, 508, 513, 501, 478, 503, 550, 536, 520, 518, 548, 506, 511, 509, 506, 541, 485, 506, 520, 512, 539, 520, 512, 518, 522, 520, 533, 494, 509, 517, 458, 508, 480, 555, 522, 506, 491, 514, 495, 528, 520, 512, 475, 525, 479, 566, 470, 505, 472, 496, 510, 520, 509, 446] payload_rows_per_dst=[11178, 10962, 10976, 10820, 10882, 11008, 10842, 10840] dup_rows_per_dst=[5350, 5656, 5388, 5480, 5352, 5646, 5476, 5216] combine_rows_per_src=[11178, 10962, 10976, 10820, 10882, 11008, 10842, 10840] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87450 payload_rows_remote=76526 dup_rows=43622 dup_rows_remote=38150 dedup_ratio=0.332809 router_tokens_per_expert=[536, 516, 520, 508, 507, 522, 502, 488, 543, 500, 498, 492, 518, 497, 511, 528, 494, 505, 510, 535, 501, 487, 501, 521, 512, 514, 473, 488, 519, 531, 536, 492, 543, 521, 482, 517, 519, 473, 490, 532, 509, 540, 537, 545, 477, 547, 522, 499, 491, 531, 464, 504, 502, 500, 513, 530, 503, 495, 561, 551, 522, 505, 509, 558, 521, 503, 525, 522, 499, 528, 489, 503, 465, 535, 509, 543, 501, 525, 531, 513, 513, 484, 481, 523, 561, 489, 527, 504, 523, 520, 524, 498, 498, 515, 522, 516, 494, 495, 516, 521, 506, 573, 511, 491, 548, 520, 492, 554, 547, 514, 510, 534, 475, 501, 519, 499, 484, 481, 488, 525, 464, 510, 470, 481, 532, 508, 521, 545] payload_rows_per_dst=[10906, 10922, 11006, 10966, 10942, 10924, 10924, 10860] dup_rows_per_dst=[5466, 5316, 5500, 5512, 5482, 5472, 5728, 5146] combine_rows_per_src=[10906, 10922, 11006, 10966, 10942, 10924, 10924, 10860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87232 payload_rows_remote=76432 dup_rows=43840 dup_rows_remote=38476 dedup_ratio=0.334473 router_tokens_per_expert=[514, 512, 501, 573, 503, 533, 559, 509, 511, 489, 509, 543, 470, 506, 512, 500, 521, 498, 509, 492, 515, 551, 496, 520, 510, 536, 490, 500, 540, 502, 531, 513, 524, 529, 511, 461, 505, 525, 531, 470, 495, 491, 470, 507, 520, 540, 491, 512, 522, 531, 504, 523, 524, 506, 486, 532, 532, 535, 479, 524, 521, 542, 490, 543, 478, 515, 480, 490, 519, 522, 508, 486, 544, 526, 498, 494, 530, 504, 564, 476, 492, 521, 525, 499, 495, 566, 509, 506, 515, 531, 523, 513, 529, 502, 563, 496, 520, 482, 518, 496, 506, 504, 500, 519, 504, 538, 474, 516, 502, 519, 529, 504, 491, 541, 495, 510, 513, 521, 505, 510, 532, 451, 529, 509, 532, 514, 484, 505] payload_rows_per_dst=[10902, 10836, 10800, 11002, 10930, 11008, 10902, 10852] dup_rows_per_dst=[5586, 5612, 5364, 5586, 5338, 5562, 5360, 5432] combine_rows_per_src=[10902, 10836, 10800, 11002, 10930, 11008, 10902, 10852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87496 payload_rows_remote=76536 dup_rows=43576 dup_rows_remote=38270 dedup_ratio=0.332458 router_tokens_per_expert=[494, 487, 506, 544, 549, 499, 535, 508, 522, 524, 502, 511, 510, 476, 512, 502, 502, 521, 493, 565, 498, 495, 490, 506, 564, 522, 509, 527, 503, 508, 553, 534, 525, 529, 554, 529, 504, 493, 515, 541, 517, 509, 488, 512, 515, 464, 495, 519, 519, 492, 506, 517, 500, 525, 517, 548, 522, 505, 517, 500, 495, 494, 506, 540, 491, 545, 501, 510, 484, 542, 516, 533, 538, 526, 473, 549, 538, 500, 493, 528, 519, 500, 524, 516, 490, 483, 505, 534, 507, 476, 489, 562, 484, 487, 530, 528, 439, 537, 493, 498, 487, 522, 505, 513, 509, 514, 520, 529, 527, 495, 503, 542, 483, 489, 510, 494, 541, 535, 537, 506, 527, 488, 498, 508, 489, 506, 486, 522] payload_rows_per_dst=[11004, 11028, 10860, 10860, 10966, 10940, 10960, 10878] dup_rows_per_dst=[5358, 5552, 5558, 5546, 5568, 5328, 5306, 5360] combine_rows_per_src=[11004, 11028, 10860, 10860, 10966, 10940, 10960, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87746 payload_rows_remote=76696 dup_rows=43326 dup_rows_remote=37960 dedup_ratio=0.330551 router_tokens_per_expert=[493, 485, 511, 502, 486, 496, 533, 475, 542, 507, 490, 518, 496, 494, 521, 535, 510, 487, 526, 497, 509, 514, 508, 545, 503, 506, 526, 489, 538, 535, 487, 532, 473, 539, 513, 525, 524, 511, 516, 512, 504, 504, 504, 502, 560, 512, 544, 506, 515, 526, 511, 530, 511, 502, 531, 492, 490, 518, 535, 508, 499, 509, 521, 510, 542, 498, 484, 497, 486, 495, 519, 517, 510, 501, 535, 537, 506, 517, 532, 532, 519, 485, 488, 526, 535, 517, 513, 533, 524, 535, 521, 517, 504, 516, 493, 528, 499, 487, 495, 489, 507, 528, 497, 507, 521, 487, 493, 525, 518, 522, 513, 514, 485, 519, 557, 537, 476, 530, 502, 484, 518, 539, 486, 537, 493, 509, 544, 503] payload_rows_per_dst=[10870, 10944, 11092, 11050, 10966, 11046, 10768, 11010] dup_rows_per_dst=[5298, 5480, 5406, 5366, 5450, 5462, 5436, 5428] combine_rows_per_src=[10870, 10944, 11092, 11050, 10966, 11046, 10768, 11010] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87554 payload_rows_remote=76252 dup_rows=43518 dup_rows_remote=37516 dedup_ratio=0.332016 router_tokens_per_expert=[491, 509, 548, 522, 518, 509, 559, 524, 481, 459, 500, 507, 522, 547, 599, 481, 520, 499, 509, 506, 522, 522, 508, 485, 526, 518, 476, 487, 499, 527, 516, 504, 493, 503, 512, 485, 543, 505, 517, 474, 531, 542, 516, 478, 520, 539, 510, 513, 539, 504, 523, 502, 486, 541, 512, 488, 483, 505, 470, 464, 523, 531, 495, 513, 502, 532, 453, 547, 492, 517, 536, 506, 501, 505, 560, 498, 520, 526, 516, 506, 509, 522, 526, 484, 491, 527, 510, 528, 522, 540, 500, 534, 492, 525, 498, 514, 475, 507, 508, 476, 540, 511, 503, 502, 531, 518, 547, 506, 534, 472, 506, 494, 495, 506, 576, 507, 521, 516, 545, 503, 507, 466, 532, 540, 527, 517, 522, 527] payload_rows_per_dst=[10976, 10864, 10946, 10826, 10914, 10810, 10916, 11302] dup_rows_per_dst=[5576, 5232, 5416, 5178, 5524, 5096, 5494, 6002] combine_rows_per_src=[10976, 10864, 10946, 10826, 10914, 10810, 10916, 11302] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627458048 dispatch_meta_bytes=458640 combine_payload_bytes=627458048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627916688 ingress_bytes=702955520 link_bytes=702955520 token_bytes=1255374736 weight_movement_bytes=75497472 total_nvlink_bytes=1330872208 weight_to_token_ratio=0.060139 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914465478e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627064832 dispatch_meta_bytes=457816 combine_payload_bytes=627064832 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627522648 ingress_bytes=853557248 link_bytes=853557248 token_bytes=1254587480 weight_movement_bytes=226492416 total_nvlink_bytes=1481079896 weight_to_token_ratio=0.180531 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082430738e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87614 payload_rows_remote=76640 dup_rows=43458 dup_rows_remote=37930 dedup_ratio=0.331558 router_tokens_per_expert=[513, 483, 499, 483, 523, 520, 522, 513, 516, 525, 576, 471, 519, 517, 496, 478, 481, 499, 527, 496, 522, 495, 519, 513, 528, 553, 548, 510, 506, 498, 454, 510, 483, 483, 516, 500, 526, 495, 505, 544, 527, 479, 506, 499, 541, 460, 506, 514, 500, 502, 525, 551, 516, 533, 541, 535, 505, 494, 485, 511, 499, 509, 523, 506, 539, 500, 496, 516, 505, 536, 541, 503, 475, 517, 532, 494, 535, 520, 508, 534, 537, 526, 503, 522, 508, 480, 487, 485, 517, 523, 542, 520, 520, 505, 515, 504, 559, 529, 525, 493, 495, 541, 487, 520, 517, 507, 489, 526, 499, 488, 513, 537, 546, 491, 510, 497, 498, 528, 530, 524, 489, 509, 539, 496, 523, 513, 537, 504] payload_rows_per_dst=[10924, 10982, 10782, 10980, 10974, 10980, 11006, 10986] dup_rows_per_dst=[5384, 5336, 5386, 5490, 5528, 5408, 5444, 5482] combine_rows_per_src=[10924, 10982, 10782, 10980, 10974, 10980, 11006, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626900992 dispatch_meta_bytes=458704 combine_payload_bytes=626900992 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627359696 ingress_bytes=777895936 link_bytes=777895936 token_bytes=1254260688 weight_movement_bytes=150994944 total_nvlink_bytes=1405255632 weight_to_token_ratio=0.120386 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864815440e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626130944 dispatch_meta_bytes=459632 combine_payload_bytes=626130944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626590576 ingress_bytes=701628416 link_bytes=701628416 token_bytes=1252721520 weight_movement_bytes=75497472 total_nvlink_bytes=1328218992 weight_to_token_ratio=0.060267 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897635932e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626982912 dispatch_meta_bytes=459224 combine_payload_bytes=626982912 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627442136 ingress_bytes=702480384 link_bytes=702480384 token_bytes=1254425048 weight_movement_bytes=75497472 total_nvlink_bytes=1329922520 weight_to_token_ratio=0.060185 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908440085e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=624656384 dispatch_meta_bytes=455072 combine_payload_bytes=624656384 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625111456 ingress_bytes=700153856 link_bytes=700153856 token_bytes=1249767840 weight_movement_bytes=75497472 total_nvlink_bytes=1325265312 weight_to_token_ratio=0.060409 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.878936435e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628293632 dispatch_meta_bytes=458624 combine_payload_bytes=628293632 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628752256 ingress_bytes=854786048 link_bytes=854786048 token_bytes=1257045888 weight_movement_bytes=226492416 total_nvlink_bytes=1483538304 weight_to_token_ratio=0.180178 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083989029e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627834880 dispatch_meta_bytes=458280 combine_payload_bytes=627834880 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628293160 ingress_bytes=778829824 link_bytes=778829824 token_bytes=1256128040 weight_movement_bytes=150994944 total_nvlink_bytes=1407122984 weight_to_token_ratio=0.120207 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876658454e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87428 payload_rows_remote=76428 dup_rows=43644 dup_rows_remote=38274 dedup_ratio=0.332977 router_tokens_per_expert=[517, 508, 514, 518, 471, 490, 503, 500, 538, 500, 513, 486, 535, 495, 527, 541, 519, 505, 488, 503, 500, 508, 529, 516, 508, 552, 504, 531, 486, 485, 525, 526, 510, 549, 528, 525, 513, 485, 511, 542, 495, 502, 466, 507, 516, 501, 561, 558, 477, 489, 512, 505, 510, 497, 510, 541, 520, 525, 494, 548, 514, 514, 543, 500, 517, 492, 541, 485, 529, 510, 509, 518, 516, 479, 505, 492, 496, 534, 499, 523, 509, 469, 497, 512, 532, 551, 483, 524, 505, 505, 505, 506, 508, 488, 522, 511, 545, 542, 521, 534, 512, 516, 524, 522, 491, 464, 542, 484, 510, 515, 530, 504, 484, 527, 518, 501, 495, 542, 542, 522, 475, 521, 510, 500, 524, 531, 508, 499] payload_rows_per_dst=[10866, 11000, 10964, 10936, 10970, 10816, 10952, 10924] dup_rows_per_dst=[5446, 5370, 5574, 5462, 5320, 5438, 5560, 5474] combine_rows_per_src=[10866, 11000, 10964, 10936, 10970, 10816, 10952, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87466 payload_rows_remote=76612 dup_rows=43606 dup_rows_remote=38166 dedup_ratio=0.332687 router_tokens_per_expert=[449, 508, 486, 546, 502, 514, 529, 492, 492, 500, 517, 533, 489, 554, 471, 523, 499, 534, 490, 517, 538, 480, 534, 524, 492, 543, 494, 532, 505, 494, 521, 506, 552, 511, 509, 533, 517, 496, 532, 503, 518, 484, 525, 515, 491, 505, 549, 536, 502, 530, 529, 495, 538, 538, 519, 497, 494, 518, 517, 524, 466, 501, 539, 512, 527, 537, 514, 492, 531, 480, 522, 505, 543, 495, 492, 519, 505, 487, 489, 510, 486, 502, 503, 518, 504, 516, 548, 515, 524, 508, 547, 508, 488, 506, 495, 523, 518, 512, 495, 511, 495, 509, 533, 547, 519, 508, 530, 550, 507, 544, 497, 480, 510, 520, 537, 486, 506, 526, 543, 502, 517, 487, 550, 463, 508, 534, 475, 475] payload_rows_per_dst=[10854, 10980, 10896, 10958, 10904, 10948, 11010, 10916] dup_rows_per_dst=[5440, 5370, 5696, 5452, 5352, 5434, 5500, 5362] combine_rows_per_src=[10854, 10980, 10896, 10958, 10904, 10948, 11010, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87782 payload_rows_remote=76874 dup_rows=43290 dup_rows_remote=37786 dedup_ratio=0.330276 router_tokens_per_expert=[525, 500, 498, 513, 527, 511, 523, 513, 515, 511, 491, 512, 516, 520, 520, 521, 552, 526, 512, 517, 518, 525, 544, 478, 533, 489, 505, 525, 456, 549, 499, 515, 478, 539, 493, 489, 538, 452, 493, 510, 542, 534, 533, 450, 518, 543, 526, 510, 526, 546, 510, 528, 519, 475, 470, 490, 522, 536, 538, 509, 521, 487, 521, 526, 510, 555, 439, 497, 533, 522, 498, 508, 479, 539, 526, 539, 530, 524, 512, 467, 508, 531, 506, 546, 483, 505, 496, 525, 538, 540, 502, 539, 478, 507, 507, 495, 522, 512, 500, 526, 527, 517, 529, 502, 510, 491, 513, 568, 474, 489, 516, 488, 518, 482, 445, 531, 524, 492, 505, 509, 481, 521, 530, 515, 512, 494, 566, 512] payload_rows_per_dst=[10974, 11054, 10922, 11036, 10962, 10908, 10958, 10968] dup_rows_per_dst=[5458, 5432, 5374, 5412, 5394, 5504, 5410, 5306] combine_rows_per_src=[10974, 11054, 10922, 11036, 10962, 10908, 10958, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87650 payload_rows_remote=76738 dup_rows=43422 dup_rows_remote=38090 dedup_ratio=0.331284 router_tokens_per_expert=[516, 492, 525, 538, 555, 518, 510, 497, 506, 519, 498, 503, 561, 542, 532, 518, 509, 493, 551, 486, 498, 528, 512, 522, 491, 527, 520, 544, 535, 503, 522, 503, 472, 514, 529, 500, 503, 474, 534, 520, 498, 542, 518, 499, 508, 510, 487, 514, 509, 533, 502, 481, 511, 502, 512, 500, 505, 471, 485, 515, 510, 525, 484, 476, 531, 540, 478, 525, 526, 543, 507, 524, 481, 485, 502, 504, 527, 489, 513, 515, 537, 505, 526, 544, 488, 498, 517, 510, 500, 488, 490, 526, 500, 506, 517, 509, 513, 519, 486, 568, 529, 492, 544, 502, 517, 519, 521, 547, 502, 467, 514, 502, 517, 515, 508, 489, 526, 551, 501, 526, 484, 519, 528, 493, 530, 525, 483, 531] payload_rows_per_dst=[11050, 10992, 10912, 10794, 10944, 10896, 11008, 11054] dup_rows_per_dst=[5610, 5496, 5332, 5248, 5436, 5426, 5476, 5398] combine_rows_per_src=[11050, 10992, 10912, 10794, 10944, 10896, 11008, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76378 dup_rows=43570 dup_rows_remote=37974 dedup_ratio=0.332413 router_tokens_per_expert=[585, 481, 509, 496, 517, 536, 537, 513, 482, 533, 537, 511, 484, 502, 478, 488, 460, 513, 536, 525, 520, 541, 513, 534, 512, 495, 509, 488, 506, 505, 523, 518, 484, 549, 496, 523, 484, 561, 498, 570, 470, 505, 500, 568, 515, 530, 515, 509, 480, 490, 511, 511, 498, 489, 480, 517, 518, 478, 506, 533, 484, 496, 512, 548, 505, 576, 555, 478, 517, 495, 532, 511, 585, 508, 516, 521, 507, 511, 483, 496, 508, 508, 522, 519, 542, 479, 524, 550, 494, 515, 484, 510, 525, 518, 476, 502, 532, 489, 481, 527, 580, 509, 525, 467, 523, 487, 502, 538, 443, 519, 513, 547, 488, 530, 468, 516, 484, 534, 490, 475, 530, 528, 499, 537, 510, 549, 517, 512] payload_rows_per_dst=[10884, 10808, 10862, 10814, 11030, 11150, 10830, 11124] dup_rows_per_dst=[5324, 5588, 5330, 5288, 5506, 5730, 5208, 5596] combine_rows_per_src=[10884, 10808, 10862, 10814, 11030, 11150, 10830, 11124] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87864 payload_rows_remote=76918 dup_rows=43208 dup_rows_remote=37744 dedup_ratio=0.329651 router_tokens_per_expert=[513, 507, 515, 479, 511, 512, 526, 468, 507, 482, 499, 493, 511, 507, 520, 509, 509, 509, 486, 520, 505, 500, 511, 509, 523, 511, 533, 553, 482, 522, 506, 511, 535, 532, 510, 543, 518, 580, 526, 528, 498, 517, 502, 518, 515, 469, 488, 522, 536, 514, 524, 517, 516, 531, 517, 509, 482, 515, 539, 521, 509, 481, 517, 522, 530, 532, 519, 528, 490, 520, 528, 505, 475, 493, 511, 512, 504, 490, 517, 511, 511, 492, 488, 523, 525, 529, 504, 503, 510, 477, 490, 538, 522, 548, 518, 475, 511, 490, 513, 538, 486, 537, 517, 543, 503, 541, 493, 516, 530, 496, 475, 516, 514, 499, 515, 534, 521, 506, 489, 494, 512, 481, 485, 559, 543, 482, 535, 544] payload_rows_per_dst=[10914, 11030, 11032, 10994, 10992, 10942, 10946, 11014] dup_rows_per_dst=[5204, 5350, 5570, 5506, 5338, 5364, 5464, 5412] combine_rows_per_src=[10914, 11030, 11032, 10994, 10992, 10942, 10946, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87338 payload_rows_remote=76316 dup_rows=43734 dup_rows_remote=38264 dedup_ratio=0.333664 router_tokens_per_expert=[514, 475, 511, 529, 557, 509, 525, 489, 513, 501, 504, 464, 546, 546, 506, 514, 508, 508, 510, 455, 495, 493, 488, 511, 503, 506, 512, 557, 481, 493, 506, 523, 533, 507, 485, 534, 487, 492, 531, 542, 516, 499, 498, 526, 539, 519, 506, 497, 550, 519, 500, 496, 489, 493, 519, 558, 508, 487, 508, 552, 509, 513, 528, 517, 501, 525, 502, 496, 497, 529, 531, 520, 469, 495, 562, 483, 484, 509, 532, 505, 527, 501, 554, 491, 538, 449, 542, 534, 491, 481, 521, 511, 487, 530, 519, 530, 527, 485, 523, 489, 535, 516, 509, 561, 506, 511, 568, 528, 521, 541, 475, 510, 495, 479, 473, 527, 516, 546, 519, 490, 503, 501, 502, 492, 577, 524, 510, 522] payload_rows_per_dst=[10912, 10812, 10830, 11022, 10900, 10878, 11036, 10948] dup_rows_per_dst=[5494, 5286, 5592, 5470, 5380, 5534, 5574, 5404] combine_rows_per_src=[10912, 10812, 10830, 11022, 10900, 10878, 11036, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87550 payload_rows_remote=76548 dup_rows=43522 dup_rows_remote=37854 dedup_ratio=0.332047 router_tokens_per_expert=[507, 557, 523, 507, 525, 506, 500, 494, 510, 544, 482, 534, 496, 542, 525, 507, 512, 506, 527, 541, 525, 505, 475, 493, 512, 508, 538, 502, 525, 516, 494, 523, 511, 534, 472, 506, 482, 480, 541, 534, 521, 495, 543, 533, 503, 472, 516, 509, 509, 498, 523, 552, 502, 542, 524, 526, 498, 506, 543, 501, 498, 520, 511, 541, 524, 529, 522, 495, 521, 526, 531, 524, 504, 563, 524, 534, 527, 520, 519, 472, 508, 519, 467, 481, 511, 500, 475, 509, 516, 522, 510, 495, 535, 518, 495, 518, 541, 506, 522, 537, 486, 499, 488, 505, 528, 507, 471, 503, 486, 520, 474, 573, 506, 469, 522, 510, 514, 549, 496, 508, 487, 499, 510, 521, 490, 473, 526, 489] payload_rows_per_dst=[11006, 11040, 10924, 11046, 11002, 10804, 10870, 10858] dup_rows_per_dst=[5512, 5364, 5380, 5542, 5668, 5354, 5422, 5280] combine_rows_per_src=[11006, 11040, 10924, 11046, 11002, 10804, 10870, 10858] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626098176 dispatch_meta_bytes=458808 combine_payload_bytes=626098176 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626556984 ingress_bytes=701595648 link_bytes=701595648 token_bytes=1252655160 weight_movement_bytes=75497472 total_nvlink_bytes=1328152632 weight_to_token_ratio=0.060270 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897220387e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=629751808 dispatch_meta_bytes=458640 combine_payload_bytes=629751808 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630210448 ingress_bytes=705249280 link_bytes=705249280 token_bytes=1259962256 weight_movement_bytes=75497472 total_nvlink_bytes=1335459728 weight_to_token_ratio=0.059920 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.943553584e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627605504 dispatch_meta_bytes=459112 combine_payload_bytes=627605504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628064616 ingress_bytes=703102976 link_bytes=703102976 token_bytes=1255670120 weight_movement_bytes=75497472 total_nvlink_bytes=1331167592 weight_to_token_ratio=0.060125 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916335428e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628637696 dispatch_meta_bytes=459312 combine_payload_bytes=628637696 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629097008 ingress_bytes=779632640 link_bytes=779632640 token_bytes=1257734704 weight_movement_bytes=150994944 total_nvlink_bytes=1408729648 weight_to_token_ratio=0.120053 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.886839291e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625688576 dispatch_meta_bytes=457408 combine_payload_bytes=625688576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626145984 ingress_bytes=701186048 link_bytes=701186048 token_bytes=1251834560 weight_movement_bytes=75497472 total_nvlink_bytes=1327332032 weight_to_token_ratio=0.060309 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.892026083e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625180672 dispatch_meta_bytes=458320 combine_payload_bytes=625180672 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625638992 ingress_bytes=700678144 link_bytes=700678144 token_bytes=1250819664 weight_movement_bytes=75497472 total_nvlink_bytes=1326317136 weight_to_token_ratio=0.060358 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.885585145e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=630112256 dispatch_meta_bytes=458648 combine_payload_bytes=630112256 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=630570904 ingress_bytes=781107200 link_bytes=781107200 token_bytes=1260683160 weight_movement_bytes=150994944 total_nvlink_bytes=1411678104 weight_to_token_ratio=0.119772 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.905538788e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627081216 dispatch_meta_bytes=457608 combine_payload_bytes=627081216 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627538824 ingress_bytes=929071104 link_bytes=929071104 token_bytes=1254620040 weight_movement_bytes=301989888 total_nvlink_bytes=1556609928 weight_to_token_ratio=0.240702 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178192937e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87280 payload_rows_remote=76306 dup_rows=43792 dup_rows_remote=38300 dedup_ratio=0.334106 router_tokens_per_expert=[509, 465, 471, 519, 522, 524, 528, 516, 510, 494, 550, 525, 485, 514, 514, 497, 522, 508, 497, 523, 533, 500, 508, 531, 485, 532, 533, 522, 474, 534, 524, 507, 509, 516, 522, 533, 476, 509, 508, 533, 541, 452, 526, 489, 513, 500, 501, 515, 494, 512, 472, 547, 509, 481, 526, 508, 504, 511, 498, 521, 517, 547, 517, 496, 490, 451, 536, 506, 501, 496, 540, 517, 525, 534, 520, 536, 506, 535, 563, 508, 525, 536, 515, 454, 507, 534, 513, 544, 514, 520, 525, 506, 506, 520, 515, 523, 507, 534, 494, 523, 475, 548, 502, 482, 484, 502, 484, 507, 548, 514, 527, 495, 505, 511, 545, 467, 524, 507, 505, 531, 487, 534, 525, 539, 520, 505, 517, 488] payload_rows_per_dst=[10942, 10974, 10774, 10946, 10910, 10958, 10870, 10906] dup_rows_per_dst=[5344, 5492, 5512, 5374, 5618, 5556, 5382, 5514] combine_rows_per_src=[10942, 10974, 10774, 10946, 10910, 10958, 10870, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87428 payload_rows_remote=76512 dup_rows=43644 dup_rows_remote=38178 dedup_ratio=0.332977 router_tokens_per_expert=[522, 541, 477, 509, 539, 496, 526, 511, 498, 530, 514, 506, 550, 540, 526, 482, 497, 556, 505, 470, 529, 477, 517, 496, 497, 490, 490, 552, 480, 530, 533, 519, 535, 492, 524, 531, 496, 525, 495, 543, 496, 517, 520, 509, 509, 522, 514, 539, 535, 524, 516, 581, 504, 529, 501, 499, 527, 522, 492, 486, 468, 454, 502, 536, 526, 504, 527, 516, 513, 520, 511, 509, 523, 521, 505, 514, 537, 545, 540, 468, 493, 524, 476, 537, 538, 515, 529, 501, 509, 504, 519, 517, 487, 468, 493, 581, 543, 523, 553, 494, 490, 511, 525, 465, 519, 524, 490, 473, 484, 494, 575, 497, 508, 492, 494, 498, 484, 505, 507, 532, 500, 495, 507, 490, 522, 508, 517, 499] payload_rows_per_dst=[11074, 10860, 11002, 10944, 11048, 10916, 10826, 10758] dup_rows_per_dst=[5460, 5416, 5532, 5408, 5510, 5466, 5494, 5358] combine_rows_per_src=[11074, 10860, 11002, 10944, 11048, 10916, 10826, 10758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87594 payload_rows_remote=76674 dup_rows=43478 dup_rows_remote=37980 dedup_ratio=0.331711 router_tokens_per_expert=[462, 555, 518, 471, 531, 525, 516, 517, 531, 502, 509, 519, 490, 492, 532, 539, 490, 542, 506, 532, 498, 520, 533, 529, 486, 481, 487, 541, 534, 501, 496, 506, 540, 546, 461, 548, 520, 536, 485, 531, 506, 517, 505, 513, 508, 525, 492, 504, 566, 505, 529, 494, 544, 522, 490, 495, 504, 505, 502, 508, 489, 531, 488, 495, 547, 508, 491, 558, 505, 507, 510, 531, 544, 479, 520, 524, 519, 505, 502, 492, 498, 538, 492, 532, 508, 467, 487, 493, 517, 515, 545, 549, 517, 517, 504, 485, 527, 535, 484, 524, 495, 537, 463, 530, 472, 531, 499, 532, 522, 529, 519, 480, 506, 528, 497, 544, 516, 484, 513, 562, 482, 542, 513, 487, 499, 525, 474, 484] payload_rows_per_dst=[10920, 10966, 11026, 10934, 10954, 10884, 10992, 10918] dup_rows_per_dst=[5498, 5484, 5448, 5568, 5276, 5444, 5366, 5394] combine_rows_per_src=[10920, 10966, 11026, 10934, 10954, 10884, 10992, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87182 payload_rows_remote=76222 dup_rows=43890 dup_rows_remote=38420 dedup_ratio=0.334854 router_tokens_per_expert=[470, 518, 500, 534, 470, 509, 501, 493, 515, 493, 501, 528, 529, 531, 498, 519, 537, 523, 510, 524, 474, 531, 526, 497, 516, 516, 525, 496, 523, 522, 527, 516, 512, 540, 532, 493, 518, 539, 571, 540, 471, 491, 495, 528, 512, 531, 512, 529, 477, 503, 516, 516, 489, 509, 549, 542, 467, 497, 527, 520, 540, 477, 496, 503, 505, 518, 488, 514, 509, 518, 519, 545, 518, 496, 499, 528, 488, 492, 525, 534, 501, 521, 498, 507, 486, 511, 485, 504, 518, 513, 471, 473, 555, 546, 499, 508, 494, 482, 529, 521, 538, 541, 552, 544, 496, 500, 504, 516, 551, 517, 493, 505, 494, 491, 518, 490, 515, 561, 504, 475, 511, 528, 534, 482, 542, 493, 509, 500] payload_rows_per_dst=[10840, 10758, 11060, 10820, 10580, 11016, 11148, 10960] dup_rows_per_dst=[5242, 5328, 5568, 5436, 5214, 5616, 6016, 5470] combine_rows_per_src=[10840, 10758, 11060, 10820, 10580, 11016, 11148, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87554 payload_rows_remote=76676 dup_rows=43518 dup_rows_remote=38302 dedup_ratio=0.332016 router_tokens_per_expert=[506, 523, 514, 529, 555, 520, 541, 526, 514, 516, 492, 506, 487, 498, 531, 522, 498, 495, 500, 517, 526, 505, 503, 537, 518, 534, 502, 507, 582, 508, 462, 526, 500, 496, 519, 525, 506, 507, 492, 481, 487, 524, 492, 503, 500, 535, 454, 526, 537, 505, 479, 500, 531, 527, 533, 501, 509, 529, 479, 498, 557, 508, 489, 530, 526, 517, 531, 540, 530, 487, 538, 500, 539, 508, 517, 514, 489, 503, 513, 504, 509, 510, 520, 524, 482, 522, 501, 518, 548, 519, 499, 508, 476, 500, 526, 479, 511, 537, 522, 471, 562, 535, 491, 515, 498, 535, 530, 523, 509, 452, 468, 503, 537, 519, 518, 548, 519, 500, 493, 523, 502, 484, 533, 516, 508, 476, 528, 514] payload_rows_per_dst=[11060, 11070, 10878, 10880, 10994, 10918, 10866, 10888] dup_rows_per_dst=[5500, 5370, 5216, 5544, 5518, 5364, 5458, 5548] combine_rows_per_src=[11060, 11070, 10878, 10880, 10994, 10918, 10866, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87714 payload_rows_remote=76658 dup_rows=43358 dup_rows_remote=37874 dedup_ratio=0.330795 router_tokens_per_expert=[527, 522, 510, 498, 509, 490, 531, 526, 509, 498, 518, 515, 498, 516, 533, 516, 512, 550, 487, 504, 525, 539, 486, 535, 511, 509, 490, 523, 540, 551, 514, 510, 493, 525, 495, 536, 524, 503, 536, 505, 474, 493, 460, 507, 508, 518, 537, 522, 466, 533, 519, 531, 553, 529, 529, 551, 523, 534, 469, 519, 518, 505, 496, 489, 469, 516, 518, 467, 536, 505, 552, 488, 498, 518, 539, 527, 554, 543, 516, 524, 491, 482, 520, 503, 539, 473, 528, 484, 498, 469, 520, 531, 513, 476, 509, 514, 490, 512, 527, 472, 516, 518, 531, 489, 504, 514, 501, 568, 494, 505, 498, 520, 512, 524, 493, 561, 470, 485, 498, 505, 558, 513, 526, 513, 506, 490, 506, 495] payload_rows_per_dst=[10942, 11070, 10916, 10996, 11056, 10834, 10932, 10968] dup_rows_per_dst=[5490, 5502, 5356, 5532, 5484, 5266, 5386, 5342] combine_rows_per_src=[10942, 11070, 10916, 10996, 11056, 10834, 10932, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76666 dup_rows=43590 dup_rows_remote=38232 dedup_ratio=0.332565 router_tokens_per_expert=[559, 509, 502, 510, 491, 539, 501, 497, 502, 531, 494, 525, 521, 525, 529, 498, 509, 516, 539, 510, 493, 567, 507, 557, 487, 513, 500, 504, 468, 522, 527, 501, 542, 523, 550, 542, 544, 547, 518, 489, 513, 513, 480, 527, 508, 547, 526, 491, 489, 499, 489, 490, 541, 465, 511, 558, 492, 483, 501, 503, 519, 524, 527, 496, 505, 509, 535, 518, 487, 525, 489, 509, 521, 502, 477, 516, 521, 523, 514, 498, 497, 496, 508, 534, 519, 520, 472, 529, 530, 496, 497, 532, 507, 503, 542, 495, 492, 524, 519, 483, 515, 510, 506, 512, 513, 487, 533, 488, 513, 476, 498, 507, 476, 508, 553, 518, 503, 508, 509, 502, 556, 525, 522, 494, 515, 505, 549, 491] payload_rows_per_dst=[10886, 10996, 11124, 10816, 10924, 10892, 10862, 10982] dup_rows_per_dst=[5580, 5444, 5596, 5358, 5374, 5462, 5290, 5486] combine_rows_per_src=[10886, 10996, 11124, 10816, 10924, 10892, 10862, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626786304 dispatch_meta_bytes=458760 combine_payload_bytes=626786304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627245064 ingress_bytes=702283776 link_bytes=702283776 token_bytes=1254031368 weight_movement_bytes=75497472 total_nvlink_bytes=1329528840 weight_to_token_ratio=0.060204 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905946819e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87820 payload_rows_remote=76830 dup_rows=43252 dup_rows_remote=37884 dedup_ratio=0.329987 router_tokens_per_expert=[515, 466, 482, 518, 523, 519, 490, 534, 537, 509, 520, 473, 546, 530, 496, 515, 544, 527, 515, 492, 531, 544, 511, 500, 486, 515, 489, 486, 522, 539, 493, 501, 530, 517, 473, 514, 493, 503, 507, 535, 531, 513, 511, 453, 532, 494, 473, 497, 526, 515, 552, 513, 494, 518, 466, 549, 500, 537, 480, 459, 534, 477, 534, 542, 484, 516, 505, 544, 505, 510, 514, 517, 516, 514, 507, 503, 485, 496, 530, 507, 497, 521, 510, 538, 503, 487, 459, 524, 503, 549, 525, 499, 509, 554, 515, 544, 505, 520, 532, 532, 529, 519, 540, 505, 478, 519, 465, 508, 474, 481, 532, 540, 505, 503, 537, 538, 537, 575, 474, 511, 516, 511, 520, 512, 505, 541, 538, 504] payload_rows_per_dst=[11022, 10980, 10820, 11026, 11008, 10998, 10990, 10976] dup_rows_per_dst=[5324, 5410, 5332, 5366, 5298, 5476, 5368, 5678] combine_rows_per_src=[11022, 10980, 10820, 11026, 11008, 10998, 10990, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625098752 dispatch_meta_bytes=458424 combine_payload_bytes=625098752 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625557176 ingress_bytes=851591168 link_bytes=851591168 token_bytes=1250655928 weight_movement_bytes=226492416 total_nvlink_bytes=1477148344 weight_to_token_ratio=0.181099 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079937472e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624410624 dispatch_meta_bytes=458568 combine_payload_bytes=624410624 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624869192 ingress_bytes=775405568 link_bytes=775405568 token_bytes=1249279816 weight_movement_bytes=150994944 total_nvlink_bytes=1400274760 weight_to_token_ratio=0.120866 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.833234068e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628113408 dispatch_meta_bytes=458616 combine_payload_bytes=628113408 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628572024 ingress_bytes=779108352 link_bytes=779108352 token_bytes=1256685432 weight_movement_bytes=150994944 total_nvlink_bytes=1407680376 weight_to_token_ratio=0.120153 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.880190581e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=459912 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628589704 ingress_bytes=703627264 link_bytes=703627264 token_bytes=1256719496 weight_movement_bytes=75497472 total_nvlink_bytes=1332216968 weight_to_token_ratio=0.060075 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922984138e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627982336 dispatch_meta_bytes=458128 combine_payload_bytes=627982336 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628440464 ingress_bytes=854474752 link_bytes=854474752 token_bytes=1256422800 weight_movement_bytes=226492416 total_nvlink_bytes=1482915216 weight_to_token_ratio=0.180268 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083594262e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628047872 dispatch_meta_bytes=459592 combine_payload_bytes=628047872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628507464 ingress_bytes=703545344 link_bytes=703545344 token_bytes=1256555336 weight_movement_bytes=75497472 total_nvlink_bytes=1332052808 weight_to_token_ratio=0.060083 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921945277e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=629391360 dispatch_meta_bytes=458856 combine_payload_bytes=629391360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629850216 ingress_bytes=704888832 link_bytes=704888832 token_bytes=1259241576 weight_movement_bytes=75497472 total_nvlink_bytes=1334739048 weight_to_token_ratio=0.059955 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.938982596e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76658 dup_rows=43534 dup_rows_remote=38060 dedup_ratio=0.332138 router_tokens_per_expert=[518, 539, 441, 495, 478, 483, 456, 557, 513, 521, 532, 493, 513, 522, 546, 523, 524, 471, 487, 506, 484, 473, 501, 526, 491, 462, 521, 541, 545, 495, 482, 485, 494, 502, 529, 521, 489, 511, 522, 492, 490, 498, 522, 481, 515, 534, 547, 534, 542, 512, 534, 522, 528, 477, 494, 494, 524, 542, 502, 529, 517, 516, 531, 492, 526, 463, 459, 540, 513, 511, 497, 496, 543, 558, 504, 504, 512, 497, 531, 518, 534, 557, 503, 476, 527, 507, 512, 487, 539, 508, 530, 511, 489, 516, 495, 486, 532, 502, 534, 530, 537, 521, 520, 512, 534, 531, 533, 536, 507, 509, 508, 551, 531, 539, 548, 508, 502, 518, 485, 510, 502, 516, 515, 514, 538, 493, 490, 520] payload_rows_per_dst=[10932, 10808, 10970, 10970, 10922, 10880, 11146, 10910] dup_rows_per_dst=[5328, 5180, 5392, 5542, 5422, 5474, 5648, 5548] combine_rows_per_src=[10932, 10808, 10970, 10970, 10922, 10880, 11146, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76566 dup_rows=43546 dup_rows_remote=38158 dedup_ratio=0.332230 router_tokens_per_expert=[471, 500, 501, 498, 501, 505, 512, 506, 482, 515, 539, 488, 526, 523, 534, 525, 539, 504, 538, 503, 502, 522, 525, 510, 507, 514, 512, 500, 520, 491, 501, 486, 516, 505, 501, 496, 519, 499, 498, 459, 533, 526, 549, 518, 528, 487, 533, 519, 497, 483, 551, 529, 497, 505, 499, 505, 501, 537, 523, 504, 548, 489, 540, 512, 518, 480, 458, 512, 512, 464, 513, 524, 484, 497, 505, 542, 509, 529, 486, 540, 571, 483, 514, 532, 524, 483, 540, 498, 508, 504, 547, 548, 507, 517, 525, 484, 527, 513, 533, 538, 521, 556, 534, 506, 505, 503, 508, 523, 522, 528, 501, 539, 506, 503, 524, 527, 498, 491, 547, 463, 507, 498, 489, 525, 517, 484, 528, 508] payload_rows_per_dst=[10910, 10960, 10970, 10830, 10856, 11008, 11028, 10964] dup_rows_per_dst=[5342, 5388, 5402, 5610, 5290, 5562, 5686, 5266] combine_rows_per_src=[10910, 10960, 10970, 10830, 10856, 11008, 11028, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87754 payload_rows_remote=76838 dup_rows=43318 dup_rows_remote=37852 dedup_ratio=0.330490 router_tokens_per_expert=[580, 509, 535, 510, 503, 491, 467, 518, 507, 503, 479, 518, 475, 536, 525, 508, 520, 503, 542, 521, 501, 509, 460, 529, 519, 507, 510, 519, 514, 491, 525, 499, 527, 519, 503, 541, 508, 506, 501, 520, 526, 493, 520, 482, 515, 514, 553, 475, 510, 491, 560, 506, 527, 517, 509, 508, 522, 532, 474, 530, 552, 495, 501, 508, 528, 487, 460, 527, 517, 489, 479, 521, 491, 463, 514, 539, 513, 511, 508, 572, 509, 536, 518, 538, 536, 532, 510, 518, 547, 556, 482, 534, 491, 478, 539, 534, 494, 505, 487, 531, 518, 517, 521, 517, 501, 496, 517, 468, 494, 536, 486, 502, 503, 518, 543, 523, 479, 525, 540, 515, 510, 530, 545, 477, 492, 510, 493, 488] payload_rows_per_dst=[10964, 10824, 11056, 11008, 10870, 11154, 10962, 10916] dup_rows_per_dst=[5364, 5266, 5350, 5446, 5368, 5592, 5466, 5466] combine_rows_per_src=[10964, 10824, 11056, 11008, 10870, 11154, 10962, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627982336 dispatch_meta_bytes=458872 combine_payload_bytes=627982336 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628441208 ingress_bytes=854474752 link_bytes=854474752 token_bytes=1256423544 weight_movement_bytes=226492416 total_nvlink_bytes=1482915960 weight_to_token_ratio=0.180268 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083594262e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76606 dup_rows=43566 dup_rows_remote=38100 dedup_ratio=0.332382 router_tokens_per_expert=[481, 540, 524, 521, 471, 540, 512, 517, 477, 482, 493, 512, 502, 569, 489, 516, 490, 461, 523, 520, 521, 491, 517, 528, 583, 487, 546, 488, 473, 523, 548, 530, 509, 491, 511, 526, 516, 512, 506, 508, 533, 490, 512, 525, 511, 507, 514, 509, 512, 527, 530, 516, 544, 557, 491, 525, 518, 520, 529, 488, 477, 514, 475, 482, 523, 528, 492, 502, 534, 511, 523, 485, 544, 572, 531, 527, 512, 507, 521, 478, 523, 519, 538, 496, 509, 482, 556, 501, 477, 482, 506, 501, 537, 508, 510, 522, 511, 493, 509, 525, 543, 490, 542, 505, 490, 491, 483, 495, 513, 518, 479, 505, 492, 554, 501, 512, 519, 546, 511, 527, 541, 537, 545, 502, 497, 494, 485, 464] payload_rows_per_dst=[10900, 10936, 10952, 10972, 11062, 10904, 10828, 10952] dup_rows_per_dst=[5466, 5466, 5434, 5654, 5474, 5430, 5356, 5286] combine_rows_per_src=[10900, 10936, 10952, 10972, 11062, 10904, 10828, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76728 dup_rows=43506 dup_rows_remote=38230 dedup_ratio=0.331924 router_tokens_per_expert=[559, 527, 515, 521, 553, 554, 523, 506, 526, 529, 515, 490, 501, 500, 527, 491, 522, 525, 505, 532, 472, 516, 506, 517, 495, 551, 475, 504, 537, 547, 575, 527, 507, 519, 530, 529, 497, 519, 518, 455, 533, 507, 488, 497, 529, 523, 516, 524, 483, 515, 503, 509, 508, 517, 538, 538, 461, 475, 508, 518, 494, 478, 514, 525, 489, 533, 496, 514, 497, 486, 483, 492, 508, 486, 505, 523, 510, 495, 515, 525, 504, 527, 558, 496, 525, 507, 533, 533, 525, 496, 514, 512, 500, 509, 510, 494, 535, 506, 513, 487, 507, 490, 497, 507, 516, 532, 477, 511, 482, 515, 532, 508, 493, 487, 539, 507, 511, 527, 515, 482, 507, 499, 502, 529, 512, 531, 527, 535] payload_rows_per_dst=[11078, 10996, 10990, 10816, 10838, 11032, 10886, 10930] dup_rows_per_dst=[5596, 5616, 5392, 5352, 5276, 5454, 5344, 5476] combine_rows_per_src=[11078, 10996, 10990, 10816, 10838, 11032, 10886, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76674 dup_rows=43582 dup_rows_remote=38278 dedup_ratio=0.332504 router_tokens_per_expert=[490, 513, 552, 501, 519, 486, 494, 540, 481, 562, 512, 474, 526, 480, 517, 513, 490, 516, 507, 469, 543, 561, 527, 490, 493, 528, 523, 549, 535, 476, 502, 502, 509, 490, 506, 489, 476, 495, 513, 492, 495, 533, 526, 506, 534, 500, 481, 515, 525, 502, 478, 476, 516, 516, 534, 537, 524, 513, 497, 513, 509, 481, 520, 527, 509, 493, 511, 492, 500, 534, 570, 526, 523, 512, 524, 486, 533, 506, 537, 539, 518, 481, 530, 527, 461, 485, 484, 506, 530, 525, 549, 542, 515, 531, 534, 520, 519, 482, 508, 475, 484, 578, 505, 498, 506, 495, 512, 531, 523, 518, 498, 500, 565, 475, 537, 533, 528, 527, 488, 519, 492, 511, 498, 522, 517, 534, 473, 553] payload_rows_per_dst=[10794, 10960, 10816, 10888, 11102, 11016, 10898, 11016] dup_rows_per_dst=[5526, 5462, 5304, 5448, 5488, 5460, 5366, 5528] combine_rows_per_src=[10794, 10960, 10816, 10888, 11102, 11016, 10898, 11016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627228672 dispatch_meta_bytes=458896 combine_payload_bytes=627228672 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=627687568 ingress_bytes=702726144 link_bytes=702726144 token_bytes=1254916240 weight_movement_bytes=75497472 total_nvlink_bytes=1330413712 weight_to_token_ratio=0.060161 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911556668e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87236 payload_rows_remote=76340 dup_rows=43836 dup_rows_remote=38470 dedup_ratio=0.334442 router_tokens_per_expert=[485, 490, 496, 537, 529, 568, 482, 537, 496, 493, 501, 488, 512, 503, 518, 490, 515, 521, 521, 522, 552, 531, 547, 547, 499, 530, 537, 496, 506, 513, 490, 460, 469, 530, 500, 499, 500, 507, 501, 553, 462, 543, 531, 519, 494, 517, 541, 477, 526, 507, 531, 515, 550, 534, 478, 519, 499, 510, 503, 468, 506, 511, 513, 510, 538, 521, 494, 506, 528, 511, 525, 508, 509, 528, 514, 515, 522, 515, 493, 515, 539, 494, 510, 541, 535, 482, 515, 515, 514, 494, 501, 497, 495, 532, 514, 490, 520, 507, 534, 520, 475, 488, 502, 496, 532, 551, 521, 483, 516, 501, 496, 489, 552, 515, 536, 512, 518, 508, 486, 489, 520, 530, 516, 479, 527, 518, 503, 551] payload_rows_per_dst=[10740, 11034, 10918, 10860, 10922, 10906, 10896, 10960] dup_rows_per_dst=[5510, 5540, 5368, 5500, 5562, 5430, 5366, 5560] combine_rows_per_src=[10740, 11034, 10918, 10860, 10922, 10906, 10896, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87656 payload_rows_remote=76626 dup_rows=43416 dup_rows_remote=37882 dedup_ratio=0.331238 router_tokens_per_expert=[510, 520, 507, 508, 459, 506, 479, 447, 524, 544, 522, 523, 486, 523, 491, 521, 507, 510, 539, 486, 526, 541, 501, 520, 505, 550, 506, 482, 512, 504, 499, 512, 519, 520, 526, 497, 490, 508, 514, 579, 529, 563, 512, 482, 524, 543, 509, 501, 502, 528, 500, 514, 481, 502, 576, 522, 475, 510, 496, 556, 524, 523, 517, 556, 557, 516, 506, 495, 538, 525, 553, 520, 545, 476, 521, 551, 534, 487, 483, 497, 492, 509, 536, 480, 491, 531, 516, 546, 510, 514, 517, 504, 498, 507, 523, 561, 516, 496, 467, 503, 492, 515, 540, 533, 486, 541, 500, 478, 491, 472, 522, 508, 505, 501, 537, 484, 492, 533, 512, 498, 469, 533, 525, 476, 508, 531, 466, 499] payload_rows_per_dst=[10866, 11012, 11090, 11030, 11070, 11010, 10852, 10726] dup_rows_per_dst=[5274, 5388, 5542, 5534, 5538, 5460, 5268, 5412] combine_rows_per_src=[10866, 11012, 11090, 11030, 11070, 11010, 10852, 10726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=629456896 dispatch_meta_bytes=458760 combine_payload_bytes=629456896 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=629915656 ingress_bytes=704954368 link_bytes=704954368 token_bytes=1259372552 weight_movement_bytes=75497472 total_nvlink_bytes=1334870024 weight_to_token_ratio=0.059948 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.939813685e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627556352 dispatch_meta_bytes=458824 combine_payload_bytes=627556352 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628015176 ingress_bytes=778551296 link_bytes=778551296 token_bytes=1255571528 weight_movement_bytes=150994944 total_nvlink_bytes=1406566472 weight_to_token_ratio=0.120260 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.873126327e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628555776 dispatch_meta_bytes=459832 combine_payload_bytes=628555776 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629015608 ingress_bytes=704053248 link_bytes=704053248 token_bytes=1257571384 weight_movement_bytes=75497472 total_nvlink_bytes=1333068856 weight_to_token_ratio=0.060034 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.928386215e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628113408 dispatch_meta_bytes=459808 combine_payload_bytes=628113408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628573216 ingress_bytes=703610880 link_bytes=703610880 token_bytes=1256686624 weight_movement_bytes=75497472 total_nvlink_bytes=1332184096 weight_to_token_ratio=0.060077 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922776366e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625377280 dispatch_meta_bytes=459240 combine_payload_bytes=625377280 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625836520 ingress_bytes=700874752 link_bytes=700874752 token_bytes=1251213800 weight_movement_bytes=75497472 total_nvlink_bytes=1326711272 weight_to_token_ratio=0.060339 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888078411e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627720192 dispatch_meta_bytes=458032 combine_payload_bytes=627720192 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628178224 ingress_bytes=854212608 link_bytes=854212608 token_bytes=1255898416 weight_movement_bytes=226492416 total_nvlink_bytes=1482390832 weight_to_token_ratio=0.180343 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083261826e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87602 payload_rows_remote=76604 dup_rows=43470 dup_rows_remote=38106 dedup_ratio=0.331650 router_tokens_per_expert=[510, 483, 492, 560, 467, 540, 550, 497, 508, 522, 509, 484, 531, 481, 542, 487, 483, 527, 530, 543, 472, 483, 554, 510, 539, 537, 510, 515, 516, 495, 519, 526, 517, 531, 537, 508, 464, 513, 527, 518, 498, 491, 512, 513, 506, 530, 482, 508, 501, 521, 512, 531, 488, 485, 521, 526, 520, 471, 514, 515, 551, 474, 545, 513, 511, 504, 510, 515, 521, 503, 528, 549, 496, 521, 482, 503, 529, 502, 497, 484, 519, 517, 498, 503, 516, 498, 523, 542, 504, 517, 507, 508, 518, 519, 520, 472, 511, 523, 512, 531, 536, 538, 517, 529, 488, 521, 511, 483, 532, 528, 472, 502, 511, 482, 512, 480, 519, 531, 513, 535, 486, 498, 506, 502, 562, 532, 517, 515] payload_rows_per_dst=[10990, 11030, 10860, 10888, 10896, 10998, 11034, 10906] dup_rows_per_dst=[5336, 5488, 5450, 5488, 5414, 5364, 5434, 5496] combine_rows_per_src=[10990, 11030, 10860, 10888, 10896, 10998, 11034, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76554 dup_rows=43654 dup_rows_remote=38174 dedup_ratio=0.333054 router_tokens_per_expert=[556, 544, 543, 504, 515, 532, 538, 515, 505, 534, 528, 510, 499, 486, 465, 499, 532, 542, 554, 471, 490, 514, 487, 521, 543, 476, 549, 501, 527, 479, 504, 482, 506, 515, 548, 484, 522, 497, 521, 514, 531, 502, 493, 484, 517, 497, 497, 492, 521, 467, 499, 538, 512, 534, 518, 530, 541, 506, 542, 532, 508, 544, 506, 497, 438, 505, 527, 515, 492, 483, 515, 520, 487, 526, 520, 515, 534, 495, 508, 526, 498, 513, 478, 535, 526, 535, 522, 521, 495, 513, 508, 526, 502, 479, 495, 513, 504, 521, 504, 493, 509, 488, 529, 512, 572, 510, 536, 501, 478, 486, 521, 561, 496, 533, 486, 492, 526, 511, 508, 543, 513, 519, 545, 512, 502, 507, 499, 494] payload_rows_per_dst=[10984, 10864, 10866, 11050, 10840, 10914, 11000, 10900] dup_rows_per_dst=[5562, 5480, 5374, 5540, 5372, 5404, 5450, 5472] combine_rows_per_src=[10984, 10864, 10866, 11050, 10840, 10914, 11000, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87564 payload_rows_remote=76586 dup_rows=43508 dup_rows_remote=38072 dedup_ratio=0.331940 router_tokens_per_expert=[516, 527, 495, 508, 511, 501, 530, 521, 510, 544, 492, 513, 531, 512, 515, 514, 460, 483, 511, 481, 474, 520, 501, 500, 529, 518, 510, 535, 527, 515, 538, 524, 523, 526, 483, 510, 508, 560, 534, 489, 521, 536, 492, 484, 495, 495, 511, 507, 514, 481, 496, 526, 502, 556, 495, 504, 469, 519, 536, 528, 564, 499, 515, 545, 520, 515, 491, 515, 495, 532, 564, 486, 543, 476, 503, 518, 473, 501, 541, 469, 496, 463, 519, 520, 511, 509, 489, 516, 516, 519, 543, 546, 535, 545, 513, 515, 533, 493, 525, 513, 502, 487, 512, 490, 522, 527, 489, 513, 516, 526, 479, 519, 550, 493, 540, 485, 517, 492, 463, 537, 514, 565, 487, 505, 527, 507, 526, 496] payload_rows_per_dst=[10986, 10868, 10968, 10828, 11124, 10992, 10820, 10978] dup_rows_per_dst=[5494, 5338, 5380, 5160, 5716, 5518, 5466, 5436] combine_rows_per_src=[10986, 10868, 10968, 10828, 11124, 10992, 10820, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=458840 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627998808 ingress_bytes=778534912 link_bytes=778534912 token_bytes=1255538776 weight_movement_bytes=150994944 total_nvlink_bytes=1406533720 weight_to_token_ratio=0.120263 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872918555e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87172 payload_rows_remote=76096 dup_rows=43900 dup_rows_remote=38128 dedup_ratio=0.334930 router_tokens_per_expert=[543, 532, 532, 500, 505, 522, 542, 486, 488, 495, 516, 497, 564, 521, 486, 517, 546, 521, 488, 501, 478, 527, 513, 485, 509, 529, 528, 522, 546, 544, 528, 554, 510, 511, 511, 535, 502, 507, 516, 512, 524, 518, 539, 498, 521, 471, 545, 521, 511, 520, 463, 464, 521, 536, 508, 518, 540, 505, 524, 474, 511, 482, 512, 544, 490, 493, 495, 487, 537, 498, 523, 538, 533, 540, 470, 538, 529, 533, 515, 496, 478, 505, 503, 486, 470, 502, 480, 498, 514, 470, 545, 575, 479, 524, 496, 529, 515, 532, 506, 486, 537, 517, 491, 544, 509, 519, 475, 479, 506, 490, 511, 513, 547, 523, 527, 492, 575, 538, 507, 466, 475, 471, 512, 500, 500, 508, 536, 521] payload_rows_per_dst=[11076, 10992, 10840, 10844, 11064, 10540, 10906, 10910] dup_rows_per_dst=[5772, 5646, 5672, 5394, 5484, 5094, 5354, 5484] combine_rows_per_src=[11076, 10992, 10840, 10844, 11064, 10540, 10906, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87340 payload_rows_remote=76426 dup_rows=43732 dup_rows_remote=38416 dedup_ratio=0.333649 router_tokens_per_expert=[525, 520, 476, 511, 520, 507, 491, 504, 532, 538, 482, 500, 496, 507, 496, 498, 545, 551, 524, 558, 554, 512, 484, 542, 524, 500, 496, 495, 511, 533, 525, 518, 517, 507, 519, 513, 519, 490, 529, 551, 485, 496, 486, 535, 492, 481, 543, 500, 496, 522, 526, 509, 526, 523, 484, 516, 512, 509, 515, 501, 517, 536, 476, 491, 495, 471, 550, 519, 518, 544, 483, 531, 501, 519, 477, 521, 509, 464, 514, 499, 516, 522, 478, 534, 541, 518, 508, 493, 509, 518, 548, 551, 511, 533, 498, 498, 550, 495, 520, 525, 536, 488, 483, 506, 511, 499, 511, 487, 543, 489, 515, 496, 514, 524, 481, 530, 483, 533, 532, 532, 529, 510, 492, 505, 530, 503, 496, 500] payload_rows_per_dst=[10846, 11036, 10820, 10812, 10914, 10970, 10980, 10962] dup_rows_per_dst=[5360, 5708, 5506, 5506, 5316, 5582, 5328, 5426] combine_rows_per_src=[10846, 11036, 10820, 10812, 10914, 10970, 10980, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87818 payload_rows_remote=76832 dup_rows=43254 dup_rows_remote=37790 dedup_ratio=0.330002 router_tokens_per_expert=[513, 471, 531, 479, 526, 518, 517, 546, 517, 483, 501, 473, 505, 481, 523, 536, 529, 498, 546, 516, 469, 494, 515, 526, 488, 499, 508, 527, 514, 517, 518, 552, 501, 506, 516, 550, 498, 542, 495, 479, 465, 536, 489, 515, 498, 490, 492, 499, 509, 507, 544, 518, 493, 484, 502, 487, 501, 535, 514, 519, 516, 479, 500, 497, 531, 553, 493, 522, 477, 516, 512, 524, 518, 505, 537, 529, 515, 530, 499, 527, 509, 544, 505, 507, 496, 551, 538, 529, 518, 527, 516, 578, 474, 487, 513, 486, 483, 494, 541, 524, 520, 510, 564, 504, 508, 491, 494, 545, 515, 509, 496, 527, 507, 533, 483, 530, 535, 529, 529, 481, 507, 507, 492, 524, 509, 498, 519, 550] payload_rows_per_dst=[10904, 11000, 10874, 10942, 10964, 11080, 10986, 11068] dup_rows_per_dst=[5336, 5432, 5268, 5268, 5612, 5476, 5464, 5398] combine_rows_per_src=[10904, 11000, 10874, 10942, 10964, 11080, 10986, 11068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87698 payload_rows_remote=76866 dup_rows=43374 dup_rows_remote=37920 dedup_ratio=0.330917 router_tokens_per_expert=[489, 507, 471, 512, 511, 470, 475, 510, 538, 517, 548, 535, 529, 521, 499, 499, 572, 518, 511, 478, 522, 534, 512, 499, 520, 496, 506, 532, 528, 474, 507, 518, 506, 498, 495, 483, 496, 504, 478, 477, 558, 536, 551, 517, 510, 522, 507, 505, 498, 525, 532, 545, 520, 530, 502, 508, 503, 527, 496, 495, 546, 522, 518, 497, 538, 549, 509, 520, 525, 510, 525, 474, 504, 477, 498, 508, 506, 506, 482, 490, 534, 532, 503, 532, 525, 546, 503, 555, 505, 513, 495, 519, 504, 493, 538, 500, 500, 514, 494, 494, 504, 532, 544, 519, 504, 491, 534, 517, 503, 529, 528, 519, 481, 560, 509, 494, 493, 485, 451, 510, 519, 486, 532, 523, 531, 508, 493, 552] payload_rows_per_dst=[10906, 10976, 10832, 11034, 10912, 11098, 10960, 10980] dup_rows_per_dst=[5356, 5478, 5454, 5494, 5330, 5496, 5492, 5274] combine_rows_per_src=[10906, 10976, 10832, 11034, 10912, 11098, 10960, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87686 payload_rows_remote=76676 dup_rows=43386 dup_rows_remote=37748 dedup_ratio=0.331009 router_tokens_per_expert=[502, 503, 477, 519, 550, 537, 545, 507, 519, 497, 482, 513, 473, 488, 493, 531, 488, 521, 529, 520, 523, 490, 525, 505, 523, 506, 493, 517, 500, 527, 478, 502, 521, 536, 493, 499, 498, 516, 488, 501, 484, 507, 500, 525, 514, 509, 494, 498, 515, 517, 497, 516, 536, 510, 552, 529, 530, 486, 543, 509, 539, 522, 513, 510, 522, 538, 479, 541, 495, 532, 512, 508, 537, 517, 513, 478, 505, 478, 484, 471, 501, 557, 535, 518, 532, 518, 497, 479, 495, 516, 525, 551, 533, 521, 523, 506, 516, 516, 514, 526, 542, 507, 525, 522, 482, 521, 504, 517, 463, 503, 530, 522, 514, 531, 502, 533, 547, 508, 522, 517, 471, 477, 552, 510, 513, 507, 505, 510] payload_rows_per_dst=[10838, 10950, 10942, 11010, 10848, 11076, 11064, 10958] dup_rows_per_dst=[5434, 5344, 5224, 5638, 5372, 5538, 5356, 5480] combine_rows_per_src=[10838, 10950, 10942, 11010, 10848, 11076, 11064, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627130368 dispatch_meta_bytes=458912 combine_payload_bytes=627130368 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627589280 ingress_bytes=778125312 link_bytes=778125312 token_bytes=1254719648 weight_movement_bytes=150994944 total_nvlink_bytes=1405714592 weight_to_token_ratio=0.120342 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.867724250e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627392512 dispatch_meta_bytes=458632 combine_payload_bytes=627392512 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627851144 ingress_bytes=778387456 link_bytes=778387456 token_bytes=1255243656 weight_movement_bytes=150994944 total_nvlink_bytes=1406238600 weight_to_token_ratio=0.120291 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.871048605e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=623378432 dispatch_meta_bytes=456896 combine_payload_bytes=623378432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623835328 ingress_bytes=698875904 link_bytes=698875904 token_bytes=1247213760 weight_movement_bytes=75497472 total_nvlink_bytes=1322711232 weight_to_token_ratio=0.060533 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.862730205e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626081792 dispatch_meta_bytes=459368 combine_payload_bytes=626081792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626541160 ingress_bytes=701579264 link_bytes=701579264 token_bytes=1252622952 weight_movement_bytes=75497472 total_nvlink_bytes=1328120424 weight_to_token_ratio=0.060272 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897012615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=629407744 dispatch_meta_bytes=458488 combine_payload_bytes=629407744 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=629866232 ingress_bytes=780402688 link_bytes=780402688 token_bytes=1259273976 weight_movement_bytes=150994944 total_nvlink_bytes=1410268920 weight_to_token_ratio=0.119906 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.896604584e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=629686272 dispatch_meta_bytes=459144 combine_payload_bytes=629686272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630145416 ingress_bytes=705183744 link_bytes=705183744 token_bytes=1259831688 weight_movement_bytes=75497472 total_nvlink_bytes=1335329160 weight_to_token_ratio=0.059927 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.942722495e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628129792 dispatch_meta_bytes=457696 combine_payload_bytes=628129792 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628587488 ingress_bytes=854622208 link_bytes=854622208 token_bytes=1256717280 weight_movement_bytes=226492416 total_nvlink_bytes=1483209696 weight_to_token_ratio=0.180225 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083781257e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87368 payload_rows_remote=76504 dup_rows=43704 dup_rows_remote=38352 dedup_ratio=0.333435 router_tokens_per_expert=[474, 541, 543, 495, 508, 512, 481, 506, 493, 518, 471, 518, 543, 514, 536, 495, 505, 481, 555, 529, 550, 498, 501, 515, 516, 486, 507, 509, 502, 507, 520, 532, 525, 503, 550, 512, 508, 531, 520, 517, 508, 519, 521, 460, 501, 494, 489, 527, 498, 516, 507, 535, 519, 556, 532, 484, 533, 509, 517, 522, 521, 513, 523, 543, 521, 500, 533, 521, 509, 525, 514, 513, 557, 516, 487, 499, 541, 507, 542, 536, 504, 517, 505, 500, 510, 528, 533, 497, 481, 530, 505, 527, 480, 503, 499, 489, 481, 475, 517, 504, 491, 471, 550, 503, 499, 497, 493, 502, 524, 502, 513, 500, 509, 525, 526, 538, 490, 534, 505, 489, 500, 551, 499, 551, 480, 510, 494, 510] payload_rows_per_dst=[10844, 10926, 10978, 11042, 11070, 10864, 10660, 10984] dup_rows_per_dst=[5452, 5500, 5392, 5614, 5572, 5352, 5384, 5438] combine_rows_per_src=[10844, 10926, 10978, 11042, 11070, 10864, 10660, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=459424 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627180192 ingress_bytes=702218240 link_bytes=702218240 token_bytes=1253900960 weight_movement_bytes=75497472 total_nvlink_bytes=1329398432 weight_to_token_ratio=0.060210 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905115730e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76322 dup_rows=43582 dup_rows_remote=37784 dedup_ratio=0.332504 router_tokens_per_expert=[518, 510, 509, 515, 467, 508, 479, 522, 479, 501, 492, 480, 527, 477, 459, 493, 547, 537, 504, 475, 519, 503, 520, 550, 508, 488, 501, 528, 525, 524, 499, 493, 503, 491, 499, 514, 523, 530, 505, 465, 482, 483, 514, 503, 544, 510, 501, 536, 551, 539, 520, 528, 488, 481, 529, 487, 495, 518, 513, 522, 516, 495, 521, 484, 465, 546, 499, 489, 541, 514, 525, 516, 477, 529, 493, 520, 522, 527, 516, 493, 527, 541, 532, 509, 540, 514, 520, 498, 499, 527, 542, 511, 546, 503, 510, 503, 537, 512, 578, 521, 512, 505, 508, 522, 477, 535, 511, 502, 536, 558, 515, 513, 518, 496, 526, 499, 543, 543, 514, 468, 525, 535, 513, 560, 464, 486, 546, 517] payload_rows_per_dst=[10558, 11122, 10866, 10902, 10700, 11038, 11136, 11168] dup_rows_per_dst=[4924, 5544, 5340, 5466, 5190, 5606, 5714, 5798] combine_rows_per_src=[10558, 11122, 10866, 10902, 10700, 11038, 11136, 11168] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76396 dup_rows=43654 dup_rows_remote=38180 dedup_ratio=0.333054 router_tokens_per_expert=[557, 520, 500, 470, 514, 517, 519, 519, 500, 523, 508, 518, 475, 519, 523, 530, 491, 522, 506, 500, 511, 541, 503, 538, 542, 540, 501, 494, 478, 532, 531, 518, 506, 510, 494, 511, 525, 524, 506, 504, 477, 475, 506, 544, 518, 460, 488, 494, 479, 481, 468, 555, 464, 476, 477, 523, 499, 513, 546, 526, 477, 539, 523, 519, 487, 512, 512, 550, 534, 529, 511, 516, 508, 528, 516, 489, 504, 478, 519, 514, 508, 509, 509, 490, 531, 509, 529, 504, 513, 501, 504, 528, 540, 538, 508, 501, 523, 503, 518, 503, 547, 522, 520, 553, 522, 532, 500, 528, 539, 512, 499, 528, 532, 533, 511, 482, 498, 504, 518, 509, 550, 512, 507, 477, 493, 540, 504, 521] payload_rows_per_dst=[10932, 11022, 10800, 10810, 10960, 10942, 11026, 10926] dup_rows_per_dst=[5492, 5474, 5284, 5320, 5454, 5502, 5672, 5456] combine_rows_per_src=[10932, 11022, 10800, 10810, 10960, 10942, 11026, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87046 payload_rows_remote=76176 dup_rows=44026 dup_rows_remote=38716 dedup_ratio=0.335892 router_tokens_per_expert=[538, 519, 517, 475, 526, 524, 500, 530, 534, 480, 540, 446, 533, 535, 547, 506, 508, 521, 499, 472, 511, 533, 508, 478, 510, 542, 500, 510, 520, 521, 506, 490, 523, 515, 535, 531, 505, 521, 502, 505, 491, 539, 514, 519, 527, 506, 509, 519, 535, 508, 520, 514, 517, 519, 538, 508, 519, 494, 527, 521, 462, 495, 518, 508, 481, 523, 533, 477, 504, 509, 508, 521, 518, 488, 513, 524, 510, 499, 489, 493, 490, 508, 517, 481, 494, 522, 497, 504, 532, 474, 485, 542, 520, 522, 507, 518, 537, 525, 498, 544, 513, 527, 546, 541, 531, 512, 495, 520, 532, 489, 507, 495, 534, 505, 493, 509, 488, 533, 523, 528, 529, 522, 489, 516, 505, 486, 493, 525] payload_rows_per_dst=[10822, 10738, 11014, 10860, 10870, 10826, 11082, 10834] dup_rows_per_dst=[5678, 5520, 5508, 5546, 5310, 5400, 5542, 5522] combine_rows_per_src=[10822, 10738, 11014, 10860, 10870, 10826, 11082, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87390 payload_rows_remote=76426 dup_rows=43682 dup_rows_remote=37906 dedup_ratio=0.333267 router_tokens_per_expert=[543, 480, 493, 497, 511, 503, 547, 515, 539, 538, 554, 544, 507, 539, 443, 480, 478, 477, 505, 532, 499, 501, 514, 491, 513, 480, 473, 493, 542, 523, 493, 514, 571, 518, 491, 529, 534, 510, 501, 503, 501, 477, 517, 499, 562, 490, 468, 494, 537, 521, 534, 499, 559, 549, 513, 516, 552, 511, 554, 509, 454, 495, 505, 536, 489, 533, 544, 491, 506, 492, 491, 484, 489, 562, 531, 525, 518, 505, 507, 554, 496, 494, 486, 496, 531, 525, 453, 519, 487, 531, 552, 547, 524, 511, 533, 497, 517, 505, 495, 534, 521, 511, 529, 552, 507, 550, 482, 491, 501, 524, 489, 474, 515, 512, 513, 504, 535, 522, 524, 495, 494, 510, 514, 526, 510, 513, 494, 500] payload_rows_per_dst=[10964, 10734, 11254, 11076, 11034, 10558, 10872, 10898] dup_rows_per_dst=[5776, 5346, 6000, 5338, 5408, 5008, 5366, 5440] combine_rows_per_src=[10964, 10734, 11254, 11076, 11034, 10558, 10872, 10898] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87622 payload_rows_remote=76650 dup_rows=43450 dup_rows_remote=37824 dedup_ratio=0.331497 router_tokens_per_expert=[531, 532, 508, 492, 540, 516, 475, 506, 508, 545, 521, 517, 502, 541, 548, 491, 472, 493, 513, 554, 537, 518, 534, 469, 497, 552, 497, 536, 491, 528, 487, 532, 494, 515, 495, 493, 513, 505, 539, 511, 503, 510, 534, 478, 521, 537, 488, 524, 485, 505, 535, 499, 537, 549, 544, 522, 515, 480, 470, 496, 525, 528, 493, 488, 544, 508, 521, 506, 543, 503, 501, 526, 513, 487, 533, 527, 519, 503, 499, 541, 499, 529, 499, 476, 515, 495, 509, 491, 506, 518, 490, 503, 495, 488, 502, 523, 496, 550, 525, 519, 514, 540, 520, 498, 513, 489, 524, 527, 535, 530, 495, 524, 482, 474, 527, 541, 499, 497, 517, 499, 508, 477, 525, 489, 497, 547, 523, 509] payload_rows_per_dst=[10984, 11006, 10948, 10988, 11010, 10842, 10972, 10872] dup_rows_per_dst=[5562, 5414, 5372, 5354, 5538, 5234, 5626, 5350] combine_rows_per_src=[10984, 11006, 10948, 10988, 11010, 10842, 10972, 10872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87466 payload_rows_remote=76476 dup_rows=43606 dup_rows_remote=38138 dedup_ratio=0.332687 router_tokens_per_expert=[543, 550, 504, 522, 483, 500, 560, 522, 517, 525, 527, 504, 484, 481, 493, 487, 472, 557, 505, 481, 529, 533, 528, 532, 517, 527, 511, 502, 511, 492, 492, 516, 502, 517, 522, 507, 496, 488, 522, 521, 487, 520, 529, 507, 520, 541, 537, 513, 513, 467, 538, 501, 520, 526, 508, 528, 526, 522, 561, 517, 530, 454, 502, 521, 507, 496, 508, 520, 530, 492, 495, 540, 517, 509, 492, 508, 506, 495, 572, 524, 532, 531, 502, 491, 498, 485, 482, 514, 505, 490, 515, 496, 535, 509, 496, 552, 505, 501, 498, 517, 489, 466, 504, 511, 510, 520, 506, 514, 513, 509, 510, 517, 521, 477, 530, 527, 498, 481, 522, 518, 514, 524, 537, 503, 517, 553, 515, 495] payload_rows_per_dst=[10940, 11048, 10990, 10902, 10984, 10842, 10848, 10912] dup_rows_per_dst=[5464, 5362, 5468, 5566, 5438, 5424, 5332, 5552] combine_rows_per_src=[10940, 11048, 10990, 10902, 10984, 10842, 10848, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87568 payload_rows_remote=76562 dup_rows=43504 dup_rows_remote=38048 dedup_ratio=0.331909 router_tokens_per_expert=[505, 517, 486, 519, 512, 522, 500, 504, 511, 544, 521, 529, 531, 546, 523, 493, 492, 515, 506, 497, 513, 497, 515, 519, 523, 490, 487, 522, 549, 505, 549, 489, 501, 535, 518, 541, 517, 510, 528, 526, 487, 510, 480, 504, 512, 551, 521, 531, 504, 522, 507, 523, 494, 508, 526, 511, 501, 537, 512, 490, 541, 514, 522, 519, 492, 544, 521, 537, 537, 494, 518, 517, 520, 545, 488, 510, 487, 519, 559, 460, 495, 545, 490, 497, 518, 532, 513, 507, 516, 547, 475, 494, 519, 495, 511, 504, 500, 506, 539, 492, 521, 548, 519, 499, 517, 489, 451, 518, 513, 469, 496, 506, 481, 491, 548, 509, 488, 519, 504, 507, 497, 515, 493, 537, 515, 506, 495, 508] payload_rows_per_dst=[10878, 10942, 11044, 11006, 11018, 10976, 10896, 10808] dup_rows_per_dst=[5648, 5394, 5500, 5456, 5478, 5340, 5270, 5418] combine_rows_per_src=[10878, 10942, 11044, 11006, 11018, 10976, 10896, 10808] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625229824 dispatch_meta_bytes=456424 combine_payload_bytes=625229824 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625686248 ingress_bytes=700727296 link_bytes=700727296 token_bytes=1250916072 weight_movement_bytes=75497472 total_nvlink_bytes=1326413544 weight_to_token_ratio=0.060354 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.886208462e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458304 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626294336 ingress_bytes=701333504 link_bytes=701333504 token_bytes=1252130368 weight_movement_bytes=75497472 total_nvlink_bytes=1327627840 weight_to_token_ratio=0.060295 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893896032e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624033792 dispatch_meta_bytes=459568 combine_payload_bytes=624033792 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624493360 ingress_bytes=775028736 link_bytes=775028736 token_bytes=1248527152 weight_movement_bytes=150994944 total_nvlink_bytes=1399522096 weight_to_token_ratio=0.120938 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.828455308e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626081792 dispatch_meta_bytes=457328 combine_payload_bytes=626081792 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626539120 ingress_bytes=701579264 link_bytes=701579264 token_bytes=1252620912 weight_movement_bytes=75497472 total_nvlink_bytes=1328118384 weight_to_token_ratio=0.060272 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897012615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=457896 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628374696 ingress_bytes=778911744 link_bytes=778911744 token_bytes=1256291496 weight_movement_bytes=150994944 total_nvlink_bytes=1407286440 weight_to_token_ratio=0.120191 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.877697315e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87372 payload_rows_remote=76366 dup_rows=43700 dup_rows_remote=38378 dedup_ratio=0.333405 router_tokens_per_expert=[469, 526, 524, 453, 476, 483, 497, 521, 528, 557, 493, 502, 523, 562, 478, 499, 483, 493, 510, 531, 527, 551, 517, 524, 534, 460, 482, 491, 512, 509, 481, 487, 521, 511, 479, 503, 520, 512, 509, 475, 451, 496, 491, 546, 526, 541, 495, 491, 541, 514, 475, 490, 492, 497, 464, 512, 526, 528, 519, 498, 483, 521, 484, 510, 502, 558, 500, 547, 518, 532, 520, 526, 470, 503, 512, 553, 535, 535, 513, 508, 505, 550, 514, 535, 499, 462, 514, 475, 500, 518, 515, 494, 531, 511, 537, 504, 502, 527, 516, 569, 514, 526, 497, 521, 544, 526, 562, 506, 540, 543, 504, 547, 482, 530, 517, 482, 539, 523, 475, 573, 535, 497, 508, 559, 508, 521, 542, 501] payload_rows_per_dst=[10836, 10842, 10890, 10758, 11052, 11006, 11046, 10942] dup_rows_per_dst=[5346, 5342, 5244, 5350, 5612, 5322, 5842, 5642] combine_rows_per_src=[10836, 10842, 10890, 10758, 11052, 11006, 11046, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626491392 dispatch_meta_bytes=458456 combine_payload_bytes=626491392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626949848 ingress_bytes=701988864 link_bytes=701988864 token_bytes=1253441240 weight_movement_bytes=75497472 total_nvlink_bytes=1328938712 weight_to_token_ratio=0.060232 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.902206920e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=458440 combine_payload_bytes=627195904 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627654344 ingress_bytes=853688320 link_bytes=853688320 token_bytes=1254850248 weight_movement_bytes=226492416 total_nvlink_bytes=1481342664 weight_to_token_ratio=0.180494 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082596955e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625590272 dispatch_meta_bytes=458976 combine_payload_bytes=625590272 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626049248 ingress_bytes=852082688 link_bytes=852082688 token_bytes=1251639520 weight_movement_bytes=226492416 total_nvlink_bytes=1478131936 weight_to_token_ratio=0.180957 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080560788e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87860 payload_rows_remote=76750 dup_rows=43212 dup_rows_remote=37632 dedup_ratio=0.329681 router_tokens_per_expert=[519, 533, 500, 526, 537, 474, 465, 486, 517, 526, 494, 522, 499, 479, 544, 497, 477, 499, 525, 463, 539, 507, 512, 506, 524, 519, 514, 495, 514, 518, 503, 529, 510, 511, 501, 540, 531, 526, 544, 520, 501, 540, 511, 537, 534, 499, 519, 508, 505, 514, 525, 578, 485, 528, 511, 513, 500, 531, 548, 550, 493, 518, 501, 531, 485, 536, 481, 469, 528, 504, 543, 521, 483, 509, 519, 542, 525, 502, 511, 494, 524, 523, 474, 495, 480, 518, 507, 526, 466, 519, 515, 510, 534, 520, 550, 495, 561, 524, 519, 541, 505, 480, 492, 501, 515, 505, 509, 491, 507, 561, 523, 505, 489, 515, 526, 512, 500, 470, 539, 475, 511, 517, 520, 497, 507, 509, 476, 501] payload_rows_per_dst=[10660, 10950, 11134, 11090, 10940, 10762, 11214, 11110] dup_rows_per_dst=[4924, 5338, 5884, 5572, 5364, 4988, 5562, 5580] combine_rows_per_src=[10660, 10950, 11134, 11090, 10940, 10762, 11214, 11110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76498 dup_rows=43506 dup_rows_remote=38006 dedup_ratio=0.331924 router_tokens_per_expert=[515, 518, 517, 532, 531, 544, 502, 514, 527, 573, 506, 513, 532, 540, 521, 484, 529, 497, 494, 491, 518, 533, 470, 480, 513, 514, 494, 501, 497, 489, 491, 492, 533, 542, 520, 499, 493, 518, 505, 477, 531, 492, 517, 497, 488, 536, 503, 513, 497, 497, 486, 498, 506, 506, 484, 507, 528, 502, 516, 485, 515, 487, 540, 511, 501, 547, 540, 484, 501, 552, 509, 499, 495, 491, 509, 503, 488, 513, 495, 518, 554, 513, 570, 542, 484, 552, 472, 526, 514, 542, 540, 497, 491, 537, 505, 504, 560, 531, 485, 509, 544, 512, 528, 508, 505, 494, 530, 488, 528, 523, 519, 505, 517, 516, 516, 529, 490, 514, 503, 488, 547, 536, 494, 502, 568, 479, 494, 485] payload_rows_per_dst=[11086, 11068, 10982, 10836, 10896, 10992, 10728, 10978] dup_rows_per_dst=[5652, 5500, 5346, 5294, 5394, 5694, 5248, 5378] combine_rows_per_src=[11086, 11068, 10982, 10836, 10896, 10992, 10728, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87570 payload_rows_remote=76704 dup_rows=43502 dup_rows_remote=37998 dedup_ratio=0.331894 router_tokens_per_expert=[511, 527, 529, 531, 520, 511, 511, 529, 534, 489, 517, 508, 467, 526, 536, 529, 504, 527, 570, 471, 505, 463, 571, 490, 488, 488, 545, 447, 512, 516, 536, 486, 501, 526, 541, 518, 528, 527, 495, 549, 515, 521, 492, 496, 490, 526, 485, 504, 475, 481, 542, 511, 501, 522, 494, 506, 486, 503, 544, 482, 517, 512, 487, 505, 510, 514, 492, 530, 510, 508, 562, 502, 479, 544, 521, 502, 498, 513, 484, 516, 547, 521, 528, 513, 495, 480, 522, 527, 492, 521, 590, 502, 519, 546, 503, 520, 525, 552, 501, 514, 513, 508, 489, 499, 492, 561, 522, 500, 518, 555, 503, 493, 498, 487, 544, 492, 546, 489, 519, 527, 488, 519, 479, 498, 501, 498, 497, 522] payload_rows_per_dst=[11038, 10948, 11016, 10818, 10866, 11112, 10930, 10842] dup_rows_per_dst=[5512, 5290, 5412, 5318, 5504, 5540, 5560, 5366] combine_rows_per_src=[11038, 10948, 11016, 10818, 10866, 11112, 10930, 10842] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87534 payload_rows_remote=76386 dup_rows=43538 dup_rows_remote=37792 dedup_ratio=0.332169 router_tokens_per_expert=[534, 525, 574, 550, 501, 479, 502, 499, 553, 560, 539, 525, 512, 502, 521, 539, 479, 472, 531, 474, 517, 508, 464, 508, 516, 489, 509, 531, 494, 494, 499, 545, 491, 483, 506, 536, 494, 510, 489, 501, 510, 521, 527, 514, 532, 511, 479, 515, 490, 519, 536, 521, 513, 535, 548, 524, 516, 540, 499, 513, 497, 566, 509, 491, 519, 481, 520, 494, 528, 513, 520, 519, 510, 507, 505, 547, 490, 541, 510, 536, 518, 496, 523, 508, 523, 488, 501, 503, 536, 500, 457, 509, 512, 523, 513, 510, 484, 522, 511, 534, 535, 523, 509, 522, 521, 522, 522, 539, 521, 491, 512, 510, 505, 487, 515, 514, 469, 489, 510, 504, 531, 495, 510, 507, 484, 522, 504, 471] payload_rows_per_dst=[11148, 10762, 10930, 10956, 11038, 11176, 10736, 10788] dup_rows_per_dst=[5746, 5298, 5638, 5284, 5442, 5708, 5176, 5246] combine_rows_per_src=[11148, 10762, 10930, 10956, 11038, 11176, 10736, 10788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87636 payload_rows_remote=76576 dup_rows=43436 dup_rows_remote=38024 dedup_ratio=0.331390 router_tokens_per_expert=[509, 517, 542, 475, 531, 509, 522, 511, 523, 504, 485, 525, 536, 523, 536, 490, 513, 541, 517, 478, 487, 507, 528, 486, 500, 456, 480, 501, 493, 491, 559, 505, 535, 519, 506, 465, 552, 497, 511, 506, 517, 539, 521, 545, 509, 501, 514, 501, 505, 489, 540, 449, 517, 507, 502, 508, 513, 503, 530, 544, 488, 536, 509, 508, 483, 476, 505, 506, 561, 531, 476, 533, 519, 533, 464, 500, 537, 509, 506, 512, 524, 510, 535, 494, 515, 520, 500, 488, 481, 492, 533, 568, 511, 485, 521, 552, 520, 484, 486, 517, 524, 530, 497, 501, 520, 518, 573, 511, 525, 491, 516, 523, 542, 521, 515, 494, 550, 512, 527, 531, 513, 501, 523, 494, 502, 517, 503, 509] payload_rows_per_dst=[11048, 10756, 11026, 10870, 10928, 10944, 11060, 11004] dup_rows_per_dst=[5428, 5328, 5450, 5426, 5374, 5514, 5412, 5504] combine_rows_per_src=[11048, 10756, 11026, 10870, 10928, 10944, 11060, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628736000 dispatch_meta_bytes=457528 combine_payload_bytes=628736000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629193528 ingress_bytes=704233472 link_bytes=704233472 token_bytes=1257929528 weight_movement_bytes=75497472 total_nvlink_bytes=1333427000 weight_to_token_ratio=0.060017 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930671709e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87460 payload_rows_remote=76550 dup_rows=43612 dup_rows_remote=38256 dedup_ratio=0.332733 router_tokens_per_expert=[530, 544, 569, 501, 494, 534, 539, 467, 524, 489, 456, 524, 520, 532, 522, 508, 496, 511, 515, 502, 544, 517, 532, 477, 460, 506, 510, 508, 508, 526, 497, 521, 545, 502, 491, 443, 490, 541, 483, 535, 518, 497, 478, 533, 524, 517, 556, 480, 514, 495, 558, 530, 532, 493, 469, 549, 473, 554, 513, 481, 533, 529, 541, 501, 493, 513, 513, 525, 519, 500, 485, 541, 517, 539, 497, 479, 500, 515, 547, 513, 520, 501, 499, 472, 525, 545, 524, 498, 514, 535, 503, 534, 526, 503, 532, 479, 519, 536, 529, 510, 521, 466, 522, 543, 505, 477, 526, 483, 499, 479, 537, 544, 551, 523, 510, 513, 486, 483, 519, 521, 476, 533, 536, 496, 479, 481, 508, 538] payload_rows_per_dst=[10974, 10856, 10910, 11026, 10980, 10870, 10968, 10876] dup_rows_per_dst=[5532, 5404, 5356, 5504, 5412, 5550, 5424, 5430] combine_rows_per_src=[10974, 10856, 10910, 11026, 10980, 10870, 10968, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87788 payload_rows_remote=76744 dup_rows=43284 dup_rows_remote=37770 dedup_ratio=0.330231 router_tokens_per_expert=[514, 544, 495, 503, 539, 491, 560, 548, 477, 510, 532, 531, 540, 530, 451, 506, 484, 500, 533, 529, 535, 504, 531, 498, 540, 548, 541, 523, 467, 515, 508, 505, 476, 490, 464, 490, 539, 488, 537, 470, 518, 547, 502, 519, 514, 488, 544, 494, 523, 530, 507, 517, 518, 534, 489, 545, 538, 508, 553, 512, 513, 504, 525, 463, 559, 501, 474, 528, 471, 474, 538, 534, 544, 528, 514, 499, 503, 493, 530, 541, 516, 518, 519, 532, 534, 534, 532, 506, 466, 481, 490, 509, 507, 516, 525, 497, 499, 497, 514, 513, 510, 526, 502, 536, 551, 518, 522, 503, 543, 499, 487, 480, 518, 458, 485, 513, 506, 531, 531, 484, 494, 486, 509, 508, 488, 513, 502, 506] payload_rows_per_dst=[11044, 11018, 10974, 11044, 10950, 10892, 10982, 10884] dup_rows_per_dst=[5498, 5504, 5186, 5514, 5512, 5472, 5418, 5180] combine_rows_per_src=[11044, 11018, 10974, 11044, 10950, 10892, 10982, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626671616 dispatch_meta_bytes=458016 combine_payload_bytes=626671616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627129632 ingress_bytes=702169088 link_bytes=702169088 token_bytes=1253801248 weight_movement_bytes=75497472 total_nvlink_bytes=1329298720 weight_to_token_ratio=0.060215 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904492414e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628359168 dispatch_meta_bytes=458808 combine_payload_bytes=628359168 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628817976 ingress_bytes=779354112 link_bytes=779354112 token_bytes=1257177144 weight_movement_bytes=150994944 total_nvlink_bytes=1408172088 weight_to_token_ratio=0.120106 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.883307164e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=456712 combine_payload_bytes=625754112 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626210824 ingress_bytes=852246528 link_bytes=852246528 token_bytes=1251964936 weight_movement_bytes=226492416 total_nvlink_bytes=1478457352 weight_to_token_ratio=0.180910 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080768560e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87286 payload_rows_remote=76468 dup_rows=43786 dup_rows_remote=38184 dedup_ratio=0.334061 router_tokens_per_expert=[533, 477, 512, 530, 537, 527, 506, 503, 466, 524, 498, 515, 460, 505, 528, 530, 511, 524, 499, 527, 527, 523, 548, 538, 536, 488, 525, 517, 513, 509, 479, 449, 479, 554, 500, 504, 503, 538, 513, 537, 519, 515, 483, 523, 528, 530, 531, 549, 551, 513, 500, 500, 513, 481, 475, 535, 466, 514, 545, 498, 557, 517, 488, 485, 543, 508, 500, 518, 501, 509, 521, 492, 516, 519, 505, 469, 504, 541, 502, 504, 515, 505, 476, 498, 520, 509, 522, 500, 539, 510, 483, 507, 530, 498, 533, 565, 485, 540, 542, 494, 499, 488, 559, 462, 488, 489, 532, 497, 535, 504, 533, 507, 514, 527, 516, 522, 519, 520, 535, 491, 484, 480, 507, 502, 545, 518, 542, 490] payload_rows_per_dst=[10938, 10930, 10988, 10886, 10884, 10818, 10952, 10890] dup_rows_per_dst=[5364, 5496, 5624, 5390, 5420, 5602, 5356, 5534] combine_rows_per_src=[10938, 10930, 10988, 10886, 10884, 10818, 10952, 10890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627310592 dispatch_meta_bytes=458400 combine_payload_bytes=627310592 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627768992 ingress_bytes=778305536 link_bytes=778305536 token_bytes=1255079584 weight_movement_bytes=150994944 total_nvlink_bytes=1406074528 weight_to_token_ratio=0.120307 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.870009744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627097600 dispatch_meta_bytes=459224 combine_payload_bytes=627097600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627556824 ingress_bytes=702595072 link_bytes=702595072 token_bytes=1254654424 weight_movement_bytes=75497472 total_nvlink_bytes=1330151896 weight_to_token_ratio=0.060174 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909894490e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628686848 dispatch_meta_bytes=458056 combine_payload_bytes=628686848 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629144904 ingress_bytes=704184320 link_bytes=704184320 token_bytes=1257831752 weight_movement_bytes=75497472 total_nvlink_bytes=1333329224 weight_to_token_ratio=0.060022 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930048392e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=458608 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626884464 ingress_bytes=852918272 link_bytes=852918272 token_bytes=1253310320 weight_movement_bytes=226492416 total_nvlink_bytes=1479802736 weight_to_token_ratio=0.180715 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081620426e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87534 payload_rows_remote=76720 dup_rows=43538 dup_rows_remote=38074 dedup_ratio=0.332169 router_tokens_per_expert=[523, 498, 520, 520, 529, 560, 515, 488, 504, 506, 520, 536, 532, 511, 520, 514, 509, 509, 535, 495, 524, 511, 498, 522, 489, 508, 492, 532, 512, 477, 511, 519, 528, 558, 493, 542, 477, 507, 506, 523, 485, 510, 522, 478, 547, 508, 508, 478, 478, 526, 504, 491, 491, 531, 513, 534, 524, 494, 499, 504, 499, 531, 524, 498, 524, 496, 545, 502, 475, 529, 507, 539, 537, 486, 516, 556, 531, 523, 546, 502, 517, 525, 551, 559, 520, 524, 521, 536, 537, 510, 522, 533, 484, 523, 579, 498, 499, 544, 472, 472, 484, 476, 507, 505, 509, 483, 508, 523, 490, 498, 492, 531, 498, 518, 508, 515, 524, 505, 439, 523, 475, 492, 515, 524, 496, 518, 500, 490] payload_rows_per_dst=[10784, 10916, 10966, 11024, 11060, 11154, 10816, 10814] dup_rows_per_dst=[5308, 5370, 5374, 5284, 5568, 5766, 5404, 5464] combine_rows_per_src=[10784, 10916, 10966, 11024, 11060, 11154, 10816, 10814] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87410 payload_rows_remote=76534 dup_rows=43662 dup_rows_remote=38046 dedup_ratio=0.333115 router_tokens_per_expert=[533, 520, 496, 499, 476, 527, 538, 489, 536, 490, 533, 497, 512, 538, 520, 499, 527, 504, 522, 501, 483, 505, 473, 502, 527, 502, 518, 486, 525, 487, 518, 499, 487, 522, 506, 533, 519, 472, 505, 501, 525, 502, 492, 524, 516, 488, 495, 501, 510, 486, 517, 494, 552, 504, 516, 514, 496, 480, 538, 507, 507, 487, 509, 501, 491, 504, 530, 530, 522, 525, 506, 514, 507, 512, 473, 556, 501, 530, 541, 504, 500, 565, 510, 496, 544, 507, 514, 536, 513, 563, 527, 520, 529, 497, 508, 505, 524, 510, 501, 500, 520, 521, 516, 528, 527, 491, 525, 501, 526, 501, 537, 505, 524, 531, 499, 488, 511, 505, 507, 525, 499, 549, 548, 512, 526, 521, 506, 484] payload_rows_per_dst=[10898, 10936, 10760, 10868, 10876, 11028, 11018, 11026] dup_rows_per_dst=[5508, 5222, 5416, 5368, 5616, 5640, 5448, 5444] combine_rows_per_src=[10898, 10936, 10760, 10868, 10876, 11028, 11018, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87674 payload_rows_remote=76766 dup_rows=43398 dup_rows_remote=37976 dedup_ratio=0.331100 router_tokens_per_expert=[499, 521, 517, 493, 547, 520, 524, 498, 519, 533, 550, 504, 528, 507, 494, 502, 503, 484, 536, 529, 482, 490, 506, 497, 461, 524, 552, 503, 530, 515, 535, 518, 512, 523, 497, 533, 520, 508, 520, 471, 503, 487, 511, 480, 505, 494, 517, 519, 511, 551, 494, 529, 543, 483, 530, 465, 537, 474, 504, 529, 513, 538, 513, 515, 488, 508, 531, 540, 537, 487, 525, 494, 487, 516, 529, 540, 541, 494, 511, 483, 545, 484, 500, 530, 486, 479, 492, 495, 480, 538, 513, 502, 498, 517, 516, 524, 506, 490, 501, 517, 478, 500, 542, 527, 509, 516, 478, 545, 485, 548, 548, 564, 528, 543, 448, 495, 515, 564, 503, 502, 527, 507, 505, 534, 487, 537, 526, 501] payload_rows_per_dst=[11002, 10908, 10906, 11034, 10996, 10868, 11028, 10932] dup_rows_per_dst=[5510, 5422, 5294, 5424, 5426, 5330, 5480, 5512] combine_rows_per_src=[11002, 10908, 10906, 11034, 10996, 10868, 11028, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87708 payload_rows_remote=76678 dup_rows=43364 dup_rows_remote=37900 dedup_ratio=0.330841 router_tokens_per_expert=[513, 507, 517, 517, 513, 517, 514, 479, 546, 533, 516, 486, 500, 502, 523, 527, 542, 472, 495, 516, 532, 486, 500, 508, 501, 476, 512, 537, 476, 501, 442, 481, 503, 490, 474, 503, 531, 509, 528, 543, 488, 515, 503, 531, 507, 495, 490, 521, 537, 497, 512, 510, 522, 511, 529, 518, 474, 529, 529, 508, 513, 485, 504, 519, 510, 504, 466, 510, 552, 519, 506, 536, 488, 528, 503, 507, 534, 522, 520, 525, 520, 530, 539, 475, 504, 474, 533, 515, 528, 516, 541, 517, 521, 525, 524, 525, 482, 522, 561, 544, 539, 491, 507, 520, 490, 536, 487, 536, 482, 545, 535, 549, 514, 509, 489, 534, 533, 521, 510, 538, 519, 487, 509, 484, 508, 496, 514, 513] payload_rows_per_dst=[11030, 11072, 10894, 11028, 10914, 11014, 10786, 10970] dup_rows_per_dst=[5464, 5856, 5368, 5624, 5374, 5400, 4892, 5386] combine_rows_per_src=[11030, 11072, 10894, 11028, 10914, 11014, 10786, 10970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87594 payload_rows_remote=76654 dup_rows=43478 dup_rows_remote=38054 dedup_ratio=0.331711 router_tokens_per_expert=[507, 521, 535, 538, 475, 540, 545, 538, 502, 506, 491, 498, 532, 540, 522, 514, 530, 547, 521, 500, 501, 503, 503, 468, 493, 528, 499, 497, 548, 527, 533, 469, 510, 517, 503, 534, 505, 495, 525, 507, 500, 478, 505, 492, 481, 511, 507, 533, 530, 528, 489, 494, 518, 534, 482, 510, 503, 525, 496, 545, 504, 501, 538, 534, 509, 490, 464, 489, 501, 520, 501, 512, 482, 508, 520, 505, 519, 524, 480, 499, 504, 509, 546, 553, 530, 505, 493, 500, 540, 510, 518, 482, 474, 549, 526, 479, 514, 480, 488, 527, 498, 544, 526, 536, 527, 541, 480, 515, 485, 535, 482, 504, 530, 524, 533, 484, 535, 536, 535, 547, 535, 526, 506, 518, 489, 500, 520, 490] payload_rows_per_dst=[10986, 10968, 10860, 11002, 10836, 10948, 10940, 11054] dup_rows_per_dst=[5622, 5366, 5346, 5460, 5210, 5488, 5424, 5562] combine_rows_per_src=[10986, 10968, 10860, 11002, 10836, 10948, 10940, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628490240 dispatch_meta_bytes=459176 combine_payload_bytes=628490240 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628949416 ingress_bytes=703987712 link_bytes=703987712 token_bytes=1257439656 weight_movement_bytes=75497472 total_nvlink_bytes=1332937128 weight_to_token_ratio=0.060041 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927555126e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87860 payload_rows_remote=76838 dup_rows=43212 dup_rows_remote=37834 dedup_ratio=0.329681 router_tokens_per_expert=[534, 499, 494, 519, 513, 508, 510, 504, 471, 530, 480, 529, 513, 524, 528, 500, 523, 491, 512, 504, 516, 498, 519, 519, 482, 492, 500, 498, 545, 504, 528, 533, 564, 470, 522, 526, 500, 505, 501, 538, 489, 522, 514, 509, 508, 502, 522, 508, 530, 528, 535, 522, 508, 504, 523, 517, 513, 503, 496, 515, 538, 550, 538, 546, 503, 504, 533, 532, 533, 509, 476, 497, 519, 521, 530, 516, 512, 500, 508, 514, 477, 469, 483, 510, 503, 522, 544, 495, 516, 545, 490, 517, 500, 500, 536, 527, 504, 508, 507, 532, 503, 519, 547, 546, 546, 496, 498, 509, 502, 487, 547, 538, 482, 494, 501, 473, 534, 491, 505, 500, 494, 508, 518, 491, 506, 490, 530, 503] payload_rows_per_dst=[10882, 10984, 11022, 11142, 11052, 10956, 11046, 10776] dup_rows_per_dst=[5430, 5344, 5378, 5590, 5362, 5312, 5532, 5264] combine_rows_per_src=[10882, 10984, 11022, 11142, 11052, 10956, 11046, 10776] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76574 dup_rows=43454 dup_rows_remote=37872 dedup_ratio=0.331528 router_tokens_per_expert=[488, 524, 511, 542, 523, 514, 508, 476, 483, 525, 510, 507, 528, 508, 474, 518, 501, 475, 476, 535, 511, 507, 518, 507, 506, 496, 496, 506, 463, 508, 534, 539, 528, 493, 505, 552, 487, 510, 488, 506, 501, 502, 534, 509, 538, 518, 532, 529, 516, 540, 509, 507, 499, 536, 512, 506, 495, 517, 541, 530, 497, 547, 511, 550, 543, 505, 464, 517, 513, 516, 542, 509, 505, 504, 503, 467, 523, 512, 565, 519, 515, 512, 500, 514, 478, 484, 506, 506, 511, 481, 513, 551, 527, 573, 506, 556, 522, 487, 529, 528, 492, 520, 508, 511, 509, 533, 499, 525, 491, 491, 516, 502, 477, 511, 516, 477, 525, 524, 515, 530, 517, 559, 478, 544, 499, 530, 480, 489] payload_rows_per_dst=[10926, 10850, 11066, 11044, 10920, 10912, 11014, 10886] dup_rows_per_dst=[5352, 5306, 5398, 5582, 5494, 5554, 5312, 5456] combine_rows_per_src=[10926, 10850, 11066, 11044, 10920, 10912, 11014, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626966528 dispatch_meta_bytes=458320 combine_payload_bytes=626966528 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627424848 ingress_bytes=702464000 link_bytes=702464000 token_bytes=1254391376 weight_movement_bytes=75497472 total_nvlink_bytes=1329888848 weight_to_token_ratio=0.060187 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908232313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628146176 dispatch_meta_bytes=458312 combine_payload_bytes=628146176 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628604488 ingress_bytes=854638592 link_bytes=854638592 token_bytes=1256750664 weight_movement_bytes=226492416 total_nvlink_bytes=1483243080 weight_to_token_ratio=0.180221 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083802034e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628867072 dispatch_meta_bytes=458968 combine_payload_bytes=628867072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629326040 ingress_bytes=704364544 link_bytes=704364544 token_bytes=1258193112 weight_movement_bytes=75497472 total_nvlink_bytes=1333690584 weight_to_token_ratio=0.060005 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.932333886e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76504 dup_rows=43570 dup_rows_remote=38066 dedup_ratio=0.332413 router_tokens_per_expert=[523, 476, 532, 533, 519, 515, 481, 523, 507, 491, 540, 500, 482, 529, 520, 542, 536, 536, 491, 543, 497, 519, 523, 506, 477, 491, 515, 494, 519, 507, 463, 497, 485, 497, 509, 522, 516, 524, 507, 459, 505, 538, 504, 511, 509, 480, 517, 538, 520, 485, 522, 506, 538, 505, 518, 536, 533, 536, 515, 493, 513, 539, 466, 537, 535, 510, 515, 518, 505, 486, 527, 515, 500, 507, 540, 496, 527, 499, 511, 488, 517, 522, 513, 521, 500, 560, 517, 506, 507, 537, 500, 506, 467, 488, 517, 573, 513, 540, 492, 502, 474, 531, 492, 491, 510, 547, 518, 538, 505, 530, 525, 523, 532, 483, 556, 486, 493, 500, 522, 560, 530, 504, 502, 491, 515, 496, 516, 479] payload_rows_per_dst=[10960, 10862, 10820, 11006, 10948, 10998, 10984, 10924] dup_rows_per_dst=[5466, 5366, 5422, 5518, 5410, 5504, 5478, 5406] combine_rows_per_src=[10960, 10862, 10820, 11006, 10948, 10998, 10984, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627949568 dispatch_meta_bytes=458832 combine_payload_bytes=627949568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628408400 ingress_bytes=703447040 link_bytes=703447040 token_bytes=1256357968 weight_movement_bytes=75497472 total_nvlink_bytes=1331855440 weight_to_token_ratio=0.060092 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920698644e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=629456896 dispatch_meta_bytes=458688 combine_payload_bytes=629456896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629915584 ingress_bytes=704954368 link_bytes=704954368 token_bytes=1259372480 weight_movement_bytes=75497472 total_nvlink_bytes=1334869952 weight_to_token_ratio=0.059948 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.939813685e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627294208 dispatch_meta_bytes=457784 combine_payload_bytes=627294208 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627751992 ingress_bytes=778289152 link_bytes=778289152 token_bytes=1255046200 weight_movement_bytes=150994944 total_nvlink_bytes=1406041144 weight_to_token_ratio=0.120310 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.869801972e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=458280 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627179048 ingress_bytes=777715712 link_bytes=777715712 token_bytes=1253899816 weight_movement_bytes=150994944 total_nvlink_bytes=1404894760 weight_to_token_ratio=0.120420 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.862529946e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76504 dup_rows=43674 dup_rows_remote=38286 dedup_ratio=0.333206 router_tokens_per_expert=[492, 508, 541, 528, 500, 506, 499, 549, 504, 537, 553, 479, 505, 500, 492, 521, 514, 509, 476, 514, 485, 505, 545, 526, 500, 490, 522, 473, 501, 542, 542, 506, 469, 498, 521, 501, 531, 503, 535, 479, 486, 554, 530, 471, 491, 505, 528, 524, 535, 516, 540, 519, 538, 529, 512, 516, 526, 465, 529, 462, 489, 507, 519, 529, 512, 538, 498, 509, 522, 549, 524, 527, 508, 477, 544, 483, 567, 525, 483, 512, 484, 486, 501, 519, 533, 528, 512, 504, 510, 529, 501, 513, 540, 525, 517, 477, 486, 564, 495, 550, 519, 484, 523, 487, 523, 537, 512, 526, 532, 521, 490, 495, 491, 518, 463, 540, 516, 505, 533, 494, 526, 511, 513, 512, 516, 498, 494, 484] payload_rows_per_dst=[10776, 10842, 10846, 10922, 11112, 11058, 10948, 10894] dup_rows_per_dst=[5294, 5458, 5406, 5540, 5444, 5658, 5486, 5388] combine_rows_per_src=[10776, 10842, 10846, 10922, 11112, 11058, 10948, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87700 payload_rows_remote=76774 dup_rows=43372 dup_rows_remote=38080 dedup_ratio=0.330902 router_tokens_per_expert=[497, 509, 492, 547, 501, 517, 439, 494, 493, 509, 528, 554, 476, 489, 528, 528, 507, 560, 521, 526, 548, 524, 531, 547, 476, 522, 504, 520, 492, 510, 495, 535, 485, 545, 501, 482, 531, 518, 489, 518, 496, 519, 484, 516, 514, 455, 568, 524, 484, 470, 499, 513, 524, 468, 509, 528, 556, 505, 482, 481, 501, 521, 548, 497, 508, 495, 511, 474, 492, 516, 517, 504, 529, 546, 507, 501, 520, 459, 508, 522, 534, 477, 494, 524, 507, 519, 560, 520, 525, 544, 509, 541, 491, 515, 531, 515, 501, 546, 464, 488, 527, 523, 502, 505, 509, 525, 530, 527, 514, 493, 513, 532, 476, 497, 505, 509, 521, 514, 516, 522, 507, 516, 527, 557, 526, 516, 511, 552] payload_rows_per_dst=[10878, 11100, 10960, 10872, 10926, 11032, 10936, 10996] dup_rows_per_dst=[5324, 5536, 5330, 5300, 5292, 5580, 5462, 5548] combine_rows_per_src=[10878, 11100, 10960, 10872, 10926, 11032, 10936, 10996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87728 payload_rows_remote=76680 dup_rows=43344 dup_rows_remote=37994 dedup_ratio=0.330688 router_tokens_per_expert=[522, 515, 549, 472, 529, 529, 485, 527, 499, 498, 548, 487, 505, 535, 481, 508, 534, 501, 532, 538, 505, 506, 502, 542, 514, 503, 473, 477, 488, 515, 531, 538, 540, 509, 492, 527, 512, 507, 492, 515, 498, 492, 496, 534, 486, 489, 556, 512, 534, 500, 516, 523, 485, 507, 460, 506, 470, 499, 522, 527, 520, 509, 504, 516, 511, 486, 462, 492, 503, 523, 534, 468, 507, 475, 539, 527, 492, 516, 540, 536, 513, 506, 490, 525, 506, 512, 499, 529, 529, 489, 520, 517, 547, 524, 510, 542, 476, 506, 559, 490, 462, 507, 545, 536, 499, 495, 480, 504, 542, 508, 521, 475, 526, 514, 504, 596, 543, 519, 534, 514, 524, 557, 550, 495, 514, 542, 470, 517] payload_rows_per_dst=[11062, 11048, 10906, 10846, 10892, 10918, 10780, 11276] dup_rows_per_dst=[5316, 5350, 5408, 5350, 5330, 5598, 5430, 5562] combine_rows_per_src=[11062, 11048, 10906, 10846, 10892, 10918, 10780, 11276] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76502 dup_rows=43520 dup_rows_remote=38008 dedup_ratio=0.332031 router_tokens_per_expert=[499, 504, 542, 483, 516, 532, 523, 538, 493, 522, 558, 459, 546, 520, 521, 525, 480, 489, 487, 510, 529, 569, 517, 525, 514, 515, 545, 476, 508, 522, 545, 529, 492, 481, 513, 508, 499, 486, 455, 485, 506, 551, 520, 508, 494, 513, 523, 532, 529, 531, 493, 508, 482, 505, 485, 516, 546, 536, 500, 554, 521, 523, 516, 509, 522, 476, 532, 499, 506, 526, 500, 499, 497, 485, 547, 526, 496, 505, 515, 483, 524, 503, 494, 491, 504, 507, 513, 535, 497, 496, 530, 508, 499, 511, 545, 524, 528, 467, 569, 525, 483, 503, 507, 499, 511, 500, 502, 519, 473, 509, 522, 534, 499, 500, 522, 479, 507, 492, 522, 556, 499, 528, 540, 512, 507, 511, 535, 520] payload_rows_per_dst=[11050, 11260, 11104, 10754, 10904, 10618, 10944, 10918] dup_rows_per_dst=[5512, 5794, 5570, 5220, 5448, 5078, 5358, 5540] combine_rows_per_src=[11050, 11260, 11104, 10754, 10904, 10618, 10944, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=459160 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627179928 ingress_bytes=777715712 link_bytes=777715712 token_bytes=1253900696 weight_movement_bytes=150994944 total_nvlink_bytes=1404895640 weight_to_token_ratio=0.120420 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.862529946e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87452 payload_rows_remote=76524 dup_rows=43620 dup_rows_remote=38156 dedup_ratio=0.332794 router_tokens_per_expert=[525, 545, 526, 526, 496, 512, 491, 492, 544, 562, 505, 509, 503, 536, 520, 522, 462, 541, 492, 522, 551, 513, 528, 540, 529, 489, 494, 542, 508, 530, 515, 477, 492, 473, 478, 491, 514, 500, 476, 573, 521, 526, 488, 476, 526, 519, 469, 510, 537, 520, 506, 469, 525, 538, 493, 482, 506, 513, 566, 522, 500, 562, 474, 482, 515, 478, 512, 505, 542, 522, 530, 489, 541, 527, 500, 471, 489, 525, 543, 523, 524, 487, 496, 503, 475, 493, 536, 536, 513, 510, 527, 517, 535, 501, 501, 533, 581, 542, 499, 495, 505, 508, 496, 495, 476, 514, 530, 518, 462, 520, 538, 517, 503, 511, 517, 503, 518, 525, 476, 525, 507, 489, 491, 510, 515, 519, 544, 514] payload_rows_per_dst=[11018, 10974, 10762, 10980, 10930, 10906, 10928, 10954] dup_rows_per_dst=[5610, 5492, 5302, 5410, 5494, 5468, 5464, 5380] combine_rows_per_src=[11018, 10974, 10762, 10980, 10930, 10906, 10928, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87320 payload_rows_remote=76478 dup_rows=43752 dup_rows_remote=38380 dedup_ratio=0.333801 router_tokens_per_expert=[503, 540, 493, 488, 482, 495, 521, 497, 505, 512, 514, 529, 470, 525, 530, 503, 469, 497, 547, 536, 533, 497, 529, 516, 531, 484, 515, 459, 544, 481, 511, 475, 496, 516, 516, 549, 533, 475, 496, 514, 517, 511, 487, 538, 503, 492, 488, 476, 495, 486, 474, 512, 548, 513, 517, 493, 498, 499, 503, 496, 518, 496, 485, 520, 537, 529, 502, 502, 542, 513, 550, 521, 525, 484, 537, 526, 515, 537, 489, 531, 540, 475, 506, 506, 506, 490, 489, 553, 514, 525, 511, 546, 517, 511, 513, 545, 523, 531, 516, 496, 495, 489, 532, 501, 494, 520, 541, 548, 485, 552, 535, 508, 538, 527, 488, 569, 504, 518, 530, 496, 527, 533, 528, 497, 483, 523, 522, 509] payload_rows_per_dst=[10870, 10978, 10842, 10732, 11078, 10958, 10844, 11018] dup_rows_per_dst=[5344, 5270, 5372, 5374, 5602, 5536, 5688, 5566] combine_rows_per_src=[10870, 10978, 10842, 10732, 11078, 10958, 10844, 11018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76612 dup_rows=43546 dup_rows_remote=38126 dedup_ratio=0.332230 router_tokens_per_expert=[522, 488, 505, 494, 506, 528, 510, 514, 560, 488, 517, 516, 527, 496, 547, 525, 516, 565, 489, 549, 491, 537, 506, 545, 542, 496, 484, 494, 534, 540, 511, 512, 473, 528, 534, 525, 524, 510, 520, 490, 503, 549, 492, 536, 522, 495, 489, 516, 517, 516, 520, 509, 515, 514, 495, 485, 500, 511, 525, 501, 530, 476, 525, 528, 509, 494, 518, 492, 475, 515, 525, 493, 488, 509, 512, 501, 526, 522, 524, 525, 484, 525, 534, 537, 467, 531, 523, 509, 492, 500, 503, 455, 468, 519, 517, 518, 512, 460, 514, 484, 490, 530, 512, 492, 540, 481, 519, 513, 541, 510, 539, 478, 515, 485, 542, 522, 489, 494, 492, 523, 539, 534, 544, 530, 497, 524, 515, 539] payload_rows_per_dst=[10956, 11006, 10906, 10914, 10894, 10940, 10846, 11064] dup_rows_per_dst=[5530, 5616, 5506, 5420, 5362, 5224, 5384, 5504] combine_rows_per_src=[10956, 11006, 10906, 10914, 10894, 10940, 10846, 11064] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628932608 dispatch_meta_bytes=459416 combine_payload_bytes=628932608 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629392024 ingress_bytes=704430080 link_bytes=704430080 token_bytes=1258324632 weight_movement_bytes=75497472 total_nvlink_bytes=1333822104 weight_to_token_ratio=0.059998 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.933164975e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628162560 dispatch_meta_bytes=458696 combine_payload_bytes=628162560 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628621256 ingress_bytes=779157504 link_bytes=779157504 token_bytes=1256783816 weight_movement_bytes=150994944 total_nvlink_bytes=1407778760 weight_to_token_ratio=0.120144 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.880813898e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87410 payload_rows_remote=76512 dup_rows=43662 dup_rows_remote=38226 dedup_ratio=0.333115 router_tokens_per_expert=[517, 527, 514, 503, 514, 478, 533, 503, 504, 497, 507, 527, 513, 498, 500, 511, 559, 520, 487, 530, 488, 498, 505, 516, 515, 509, 518, 530, 500, 505, 501, 528, 470, 534, 531, 504, 502, 470, 534, 521, 531, 512, 535, 503, 493, 461, 538, 518, 515, 516, 554, 531, 509, 540, 561, 507, 519, 516, 488, 533, 503, 511, 550, 529, 500, 495, 499, 534, 547, 546, 471, 543, 553, 499, 500, 504, 512, 508, 532, 525, 495, 496, 530, 517, 490, 536, 534, 524, 494, 482, 522, 512, 507, 493, 515, 520, 532, 460, 516, 510, 502, 518, 542, 497, 477, 509, 495, 494, 540, 510, 493, 489, 500, 502, 527, 482, 525, 496, 465, 496, 494, 489, 506, 562, 527, 518, 515, 519] payload_rows_per_dst=[10890, 10882, 11074, 11022, 11030, 10898, 10776, 10838] dup_rows_per_dst=[5402, 5536, 5240, 5742, 5506, 5436, 5392, 5408] combine_rows_per_src=[10890, 10882, 11074, 11022, 11030, 10898, 10776, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626704384 dispatch_meta_bytes=458040 combine_payload_bytes=626704384 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=627162424 ingress_bytes=928694272 link_bytes=928694272 token_bytes=1253866808 weight_movement_bytes=301989888 total_nvlink_bytes=1555856696 weight_to_token_ratio=0.240847 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.038603 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177715060e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626884608 dispatch_meta_bytes=458720 combine_payload_bytes=626884608 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627343328 ingress_bytes=853377024 link_bytes=853377024 token_bytes=1254227936 weight_movement_bytes=226492416 total_nvlink_bytes=1480720352 weight_to_token_ratio=0.180583 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082202188e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626507776 dispatch_meta_bytes=459432 combine_payload_bytes=626507776 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626967208 ingress_bytes=702005248 link_bytes=702005248 token_bytes=1253474984 weight_movement_bytes=75497472 total_nvlink_bytes=1328972456 weight_to_token_ratio=0.060231 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.902414692e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627605504 dispatch_meta_bytes=458952 combine_payload_bytes=627605504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628064456 ingress_bytes=703102976 link_bytes=703102976 token_bytes=1255669960 weight_movement_bytes=75497472 total_nvlink_bytes=1331167432 weight_to_token_ratio=0.060125 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916335428e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87336 payload_rows_remote=76410 dup_rows=43736 dup_rows_remote=38312 dedup_ratio=0.333679 router_tokens_per_expert=[500, 495, 522, 499, 513, 519, 530, 529, 484, 546, 552, 521, 539, 487, 494, 472, 548, 507, 479, 475, 527, 489, 484, 527, 510, 516, 480, 543, 527, 482, 507, 499, 497, 520, 514, 552, 504, 535, 529, 491, 514, 553, 479, 560, 521, 502, 488, 548, 479, 544, 568, 517, 486, 506, 544, 478, 504, 506, 503, 504, 540, 468, 530, 501, 521, 510, 527, 542, 484, 484, 500, 521, 521, 529, 507, 515, 453, 517, 499, 486, 507, 515, 527, 544, 512, 472, 515, 527, 492, 517, 530, 482, 520, 519, 523, 468, 536, 577, 514, 511, 481, 553, 505, 503, 505, 518, 518, 477, 540, 504, 532, 535, 529, 507, 495, 521, 529, 499, 573, 529, 504, 478, 497, 497, 504, 506, 507, 479] payload_rows_per_dst=[10938, 10872, 10912, 10624, 10870, 11212, 10982, 10926] dup_rows_per_dst=[5466, 5328, 5702, 5122, 5378, 5714, 5602, 5424] combine_rows_per_src=[10938, 10872, 10912, 10624, 10870, 11212, 10982, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626786304 dispatch_meta_bytes=458952 combine_payload_bytes=626786304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627245256 ingress_bytes=702283776 link_bytes=702283776 token_bytes=1254031560 weight_movement_bytes=75497472 total_nvlink_bytes=1329529032 weight_to_token_ratio=0.060204 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905946819e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625950720 dispatch_meta_bytes=458888 combine_payload_bytes=625950720 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626409608 ingress_bytes=776945664 link_bytes=776945664 token_bytes=1252360328 weight_movement_bytes=150994944 total_nvlink_bytes=1403355272 weight_to_token_ratio=0.120568 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.852764653e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87624 payload_rows_remote=76656 dup_rows=43448 dup_rows_remote=37884 dedup_ratio=0.331482 router_tokens_per_expert=[517, 551, 492, 461, 474, 523, 523, 463, 512, 509, 469, 508, 527, 513, 523, 494, 520, 551, 521, 552, 498, 516, 536, 531, 550, 519, 476, 507, 503, 496, 530, 548, 510, 504, 496, 500, 567, 508, 493, 507, 487, 522, 569, 507, 512, 506, 528, 532, 536, 514, 495, 510, 509, 536, 532, 493, 483, 525, 477, 479, 521, 537, 522, 481, 529, 506, 494, 535, 505, 506, 515, 506, 535, 522, 537, 479, 509, 541, 540, 507, 475, 517, 540, 534, 542, 470, 468, 511, 532, 508, 506, 548, 506, 524, 496, 530, 496, 496, 554, 492, 540, 530, 507, 511, 507, 476, 486, 502, 493, 524, 523, 496, 495, 508, 501, 508, 539, 525, 507, 502, 536, 512, 496, 493, 459, 516, 502, 520] payload_rows_per_dst=[10874, 11140, 11064, 10870, 10968, 10968, 10878, 10862] dup_rows_per_dst=[5244, 5568, 5432, 5430, 5564, 5446, 5388, 5376] combine_rows_per_src=[10874, 11140, 11064, 10870, 10968, 10968, 10878, 10862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87202 payload_rows_remote=76338 dup_rows=43870 dup_rows_remote=38534 dedup_ratio=0.334702 router_tokens_per_expert=[515, 487, 550, 534, 530, 455, 526, 518, 503, 539, 520, 520, 494, 513, 483, 523, 518, 460, 497, 515, 492, 474, 514, 497, 460, 500, 518, 585, 512, 518, 554, 486, 482, 510, 535, 502, 551, 507, 524, 496, 551, 506, 507, 546, 503, 528, 500, 490, 535, 499, 501, 497, 541, 551, 493, 534, 519, 508, 517, 557, 527, 526, 480, 541, 505, 557, 515, 516, 479, 515, 499, 505, 493, 518, 549, 518, 495, 525, 516, 530, 507, 500, 518, 516, 542, 486, 527, 488, 511, 495, 492, 463, 488, 483, 471, 552, 539, 492, 491, 495, 539, 497, 485, 478, 579, 534, 477, 523, 489, 503, 524, 505, 499, 540, 500, 499, 501, 505, 508, 524, 535, 508, 509, 551, 538, 503, 516, 502] payload_rows_per_dst=[10808, 10864, 10926, 10928, 10958, 10846, 10958, 10914] dup_rows_per_dst=[5612, 5336, 5550, 5724, 5512, 5232, 5342, 5562] combine_rows_per_src=[10808, 10864, 10926, 10928, 10958, 10846, 10958, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76426 dup_rows=43590 dup_rows_remote=37978 dedup_ratio=0.332565 router_tokens_per_expert=[478, 479, 523, 505, 546, 519, 530, 553, 501, 522, 541, 521, 534, 508, 488, 501, 491, 480, 539, 553, 504, 479, 508, 512, 525, 484, 526, 500, 497, 503, 530, 534, 544, 526, 522, 449, 537, 538, 510, 541, 511, 546, 517, 504, 483, 527, 526, 503, 511, 495, 494, 496, 467, 538, 520, 485, 501, 509, 519, 519, 524, 516, 520, 518, 514, 503, 521, 513, 522, 523, 512, 516, 492, 502, 519, 517, 507, 463, 511, 501, 509, 494, 473, 523, 513, 529, 524, 505, 489, 535, 516, 530, 521, 520, 441, 537, 528, 529, 484, 543, 527, 547, 510, 555, 517, 515, 512, 488, 452, 509, 516, 525, 522, 516, 539, 522, 515, 504, 510, 495, 504, 462, 523, 503, 509, 505, 523, 502] payload_rows_per_dst=[11056, 10852, 11342, 11064, 10932, 10654, 10964, 10618] dup_rows_per_dst=[5612, 5322, 5942, 5824, 5426, 5040, 5450, 4974] combine_rows_per_src=[11056, 10852, 11342, 11064, 10932, 10654, 10964, 10618] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87142 payload_rows_remote=76148 dup_rows=43930 dup_rows_remote=38472 dedup_ratio=0.335159 router_tokens_per_expert=[532, 517, 484, 515, 480, 511, 547, 519, 542, 518, 531, 541, 556, 522, 514, 534, 528, 500, 474, 455, 522, 496, 508, 492, 505, 491, 482, 494, 562, 515, 510, 540, 527, 521, 488, 480, 522, 467, 488, 511, 526, 474, 478, 514, 514, 511, 517, 489, 498, 515, 489, 522, 496, 541, 553, 485, 512, 476, 498, 491, 536, 506, 498, 552, 487, 548, 539, 543, 513, 475, 480, 514, 507, 514, 505, 528, 513, 512, 515, 534, 511, 503, 511, 505, 503, 543, 522, 499, 543, 539, 481, 517, 498, 496, 528, 516, 492, 520, 531, 518, 511, 502, 510, 501, 525, 481, 543, 512, 506, 533, 552, 489, 512, 565, 516, 522, 537, 492, 526, 542, 504, 494, 536, 487, 516, 504, 508, 475] payload_rows_per_dst=[11084, 10862, 10780, 10812, 10812, 10926, 10994, 10872] dup_rows_per_dst=[5642, 5286, 5274, 5524, 5642, 5504, 5458, 5600] combine_rows_per_src=[11084, 10862, 10780, 10812, 10812, 10926, 10994, 10872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87202 payload_rows_remote=76344 dup_rows=43870 dup_rows_remote=38482 dedup_ratio=0.334702 router_tokens_per_expert=[530, 504, 504, 548, 490, 513, 465, 522, 477, 501, 520, 489, 539, 531, 482, 540, 511, 491, 524, 511, 525, 523, 478, 528, 535, 519, 532, 497, 506, 511, 493, 485, 495, 531, 515, 522, 555, 497, 486, 515, 508, 494, 489, 510, 535, 501, 472, 498, 551, 471, 473, 513, 535, 526, 503, 519, 531, 515, 518, 521, 485, 541, 527, 515, 512, 532, 488, 508, 520, 516, 507, 497, 512, 493, 507, 526, 539, 522, 487, 543, 481, 550, 529, 491, 546, 547, 490, 501, 503, 529, 517, 502, 510, 482, 513, 472, 533, 509, 481, 486, 537, 541, 529, 512, 492, 531, 533, 530, 525, 480, 557, 467, 511, 551, 509, 519, 551, 502, 500, 506, 518, 501, 556, 522, 509, 519, 468, 488] payload_rows_per_dst=[10920, 10870, 10858, 10938, 10822, 10908, 10972, 10914] dup_rows_per_dst=[5390, 5468, 5388, 5550, 5596, 5418, 5514, 5546] combine_rows_per_src=[10920, 10870, 10858, 10938, 10822, 10908, 10972, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87592 payload_rows_remote=76636 dup_rows=43480 dup_rows_remote=37984 dedup_ratio=0.331726 router_tokens_per_expert=[514, 484, 517, 527, 522, 524, 518, 523, 559, 528, 489, 490, 562, 527, 487, 534, 488, 505, 514, 513, 521, 525, 493, 484, 473, 483, 532, 482, 512, 495, 490, 523, 479, 488, 514, 522, 494, 553, 515, 477, 520, 540, 534, 473, 494, 504, 515, 489, 535, 508, 474, 526, 522, 520, 511, 540, 519, 555, 512, 487, 502, 490, 524, 501, 523, 525, 505, 518, 538, 502, 517, 499, 504, 525, 482, 501, 491, 483, 489, 519, 464, 518, 488, 551, 526, 495, 521, 500, 515, 493, 476, 539, 535, 549, 535, 503, 507, 557, 496, 515, 485, 530, 488, 543, 545, 543, 493, 536, 467, 511, 468, 517, 529, 484, 508, 525, 513, 521, 541, 559, 519, 536, 513, 493, 562, 508, 496, 524] payload_rows_per_dst=[11144, 10832, 10854, 10956, 10902, 10924, 11032, 10948] dup_rows_per_dst=[5466, 5234, 5368, 5496, 5340, 5492, 5370, 5714] combine_rows_per_src=[11144, 10832, 10854, 10956, 10902, 10924, 11032, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87600 payload_rows_remote=76658 dup_rows=43472 dup_rows_remote=38016 dedup_ratio=0.331665 router_tokens_per_expert=[543, 515, 528, 536, 509, 507, 517, 554, 521, 489, 495, 519, 520, 506, 495, 489, 517, 476, 490, 519, 521, 521, 485, 569, 561, 536, 508, 483, 470, 548, 517, 502, 495, 504, 539, 499, 515, 524, 506, 487, 509, 524, 552, 505, 530, 487, 520, 465, 505, 490, 513, 490, 536, 489, 482, 531, 513, 535, 514, 490, 476, 533, 534, 540, 527, 488, 505, 509, 510, 466, 539, 493, 528, 507, 514, 496, 543, 498, 542, 526, 503, 526, 515, 530, 502, 516, 503, 554, 490, 496, 535, 516, 522, 465, 546, 480, 478, 504, 522, 502, 492, 492, 530, 532, 525, 535, 496, 485, 526, 518, 542, 547, 513, 462, 538, 513, 480, 541, 479, 506, 535, 507, 506, 529, 489, 517, 522, 485] payload_rows_per_dst=[11070, 10950, 11032, 10892, 10874, 10942, 10918, 10922] dup_rows_per_dst=[5416, 5496, 5290, 5450, 5508, 5456, 5534, 5322] combine_rows_per_src=[11070, 10950, 11032, 10892, 10874, 10942, 10918, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625360896 dispatch_meta_bytes=459488 combine_payload_bytes=625360896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625820384 ingress_bytes=700858368 link_bytes=700858368 token_bytes=1251181280 weight_movement_bytes=75497472 total_nvlink_bytes=1326678752 weight_to_token_ratio=0.060341 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.887870639e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627965952 dispatch_meta_bytes=458160 combine_payload_bytes=627965952 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628424112 ingress_bytes=778960896 link_bytes=778960896 token_bytes=1256390064 weight_movement_bytes=150994944 total_nvlink_bytes=1407385008 weight_to_token_ratio=0.120182 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.878320632e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626081792 dispatch_meta_bytes=457616 combine_payload_bytes=626081792 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626539408 ingress_bytes=777076736 link_bytes=777076736 token_bytes=1252621200 weight_movement_bytes=150994944 total_nvlink_bytes=1403616144 weight_to_token_ratio=0.120543 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.854426831e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=623804416 dispatch_meta_bytes=458480 combine_payload_bytes=623804416 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=624262896 ingress_bytes=850296832 link_bytes=850296832 token_bytes=1248067312 weight_movement_bytes=226492416 total_nvlink_bytes=1474559728 weight_to_token_ratio=0.181475 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078296071e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625410048 dispatch_meta_bytes=459304 combine_payload_bytes=625410048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625869352 ingress_bytes=700907520 link_bytes=700907520 token_bytes=1251279400 weight_movement_bytes=75497472 total_nvlink_bytes=1326776872 weight_to_token_ratio=0.060336 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.888493956e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87360 payload_rows_remote=76372 dup_rows=43712 dup_rows_remote=38206 dedup_ratio=0.333496 router_tokens_per_expert=[520, 458, 516, 513, 523, 559, 548, 481, 480, 484, 526, 511, 503, 532, 498, 537, 513, 544, 516, 517, 508, 546, 536, 542, 528, 544, 517, 526, 512, 507, 478, 515, 536, 493, 529, 494, 510, 531, 511, 494, 498, 535, 509, 490, 538, 513, 532, 505, 513, 477, 442, 507, 504, 481, 525, 521, 532, 490, 495, 537, 532, 527, 512, 484, 545, 511, 526, 518, 498, 566, 518, 503, 503, 491, 547, 521, 532, 500, 475, 525, 519, 484, 495, 511, 487, 548, 488, 518, 516, 507, 501, 480, 555, 462, 492, 506, 481, 488, 550, 539, 536, 486, 469, 509, 488, 509, 490, 504, 513, 513, 542, 504, 508, 520, 484, 522, 490, 503, 521, 449, 530, 506, 555, 514, 541, 543, 543, 503] payload_rows_per_dst=[10672, 10930, 10912, 10918, 11064, 10816, 11060, 10988] dup_rows_per_dst=[5092, 5490, 5494, 5446, 5566, 5322, 5796, 5506] combine_rows_per_src=[10672, 10930, 10912, 10918, 11064, 10816, 11060, 10988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=458480 combine_payload_bytes=627802112 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628260592 ingress_bytes=854294528 link_bytes=854294528 token_bytes=1256062704 weight_movement_bytes=226492416 total_nvlink_bytes=1482555120 weight_to_token_ratio=0.180319 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083365713e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627982336 dispatch_meta_bytes=458696 combine_payload_bytes=627982336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628441032 ingress_bytes=703479808 link_bytes=703479808 token_bytes=1256423368 weight_movement_bytes=75497472 total_nvlink_bytes=1331920840 weight_to_token_ratio=0.060089 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921114188e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625639424 dispatch_meta_bytes=458312 combine_payload_bytes=625639424 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626097736 ingress_bytes=852131840 link_bytes=852131840 token_bytes=1251737160 weight_movement_bytes=226492416 total_nvlink_bytes=1478229576 weight_to_token_ratio=0.180942 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080623120e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15056141728 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=7530823072 ingress_bytes=9186263040 link_bytes=9186263040 total_nvlink_bytes=16717086112 weight_to_token_ratio=0.110317 prefetch_share_of_ingress=0.090404 dedup_ratio=0.332534 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043505 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.707895499e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87316 payload_rows_remote=76370 dup_rows=43756 dup_rows_remote=38362 dedup_ratio=0.333832 router_tokens_per_expert=[508, 526, 513, 510, 498, 520, 516, 523, 510, 534, 536, 512, 544, 515, 497, 496, 516, 489, 529, 507, 513, 509, 510, 528, 526, 505, 490, 497, 511, 505, 485, 555, 498, 564, 507, 541, 502, 508, 527, 509, 531, 537, 483, 474, 516, 512, 482, 522, 483, 524, 507, 494, 507, 530, 531, 522, 504, 525, 494, 526, 535, 512, 551, 513, 484, 501, 472, 508, 513, 525, 471, 553, 521, 475, 545, 526, 528, 523, 533, 492, 501, 484, 554, 522, 553, 505, 514, 482, 474, 511, 510, 479, 562, 487, 479, 493, 489, 545, 494, 516, 527, 499, 543, 506, 506, 543, 518, 524, 493, 505, 507, 484, 462, 515, 504, 538, 517, 504, 500, 469, 477, 515, 509, 504, 544, 554, 512, 529] payload_rows_per_dst=[10966, 10930, 10952, 10860, 10946, 10892, 10918, 10852] dup_rows_per_dst=[5550, 5420, 5474, 5656, 5394, 5328, 5480, 5454] combine_rows_per_src=[10966, 10930, 10952, 10860, 10946, 10892, 10918, 10852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76610 dup_rows=43598 dup_rows_remote=38236 dedup_ratio=0.332626 router_tokens_per_expert=[504, 559, 516, 552, 535, 473, 514, 507, 488, 506, 542, 489, 521, 558, 513, 463, 532, 517, 510, 485, 537, 507, 540, 472, 508, 527, 504, 537, 478, 446, 494, 519, 509, 509, 520, 524, 528, 507, 527, 511, 529, 479, 509, 468, 520, 518, 478, 522, 523, 511, 504, 495, 525, 486, 557, 525, 510, 511, 519, 551, 505, 522, 539, 523, 535, 473, 514, 533, 499, 506, 500, 469, 525, 533, 540, 503, 527, 531, 467, 504, 508, 505, 512, 475, 516, 494, 525, 504, 512, 519, 536, 549, 496, 523, 500, 498, 529, 491, 475, 526, 515, 514, 538, 517, 504, 509, 518, 506, 556, 496, 499, 481, 524, 484, 505, 507, 498, 543, 532, 509, 506, 528, 513, 509, 523, 519, 507, 507] payload_rows_per_dst=[10902, 10864, 10912, 10978, 11020, 10950, 10938, 10910] dup_rows_per_dst=[5578, 5362, 5404, 5634, 5298, 5394, 5410, 5518] combine_rows_per_src=[10902, 10864, 10912, 10978, 11020, 10950, 10938, 10910] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15046499048 prefetch_bytes=830472192 reduce_bytes=528482304 weight_movement_bytes=1358954496 egress_bytes=7525997288 ingress_bytes=8879456256 link_bytes=8879456256 total_nvlink_bytes=16405453544 weight_to_token_ratio=0.090317 prefetch_share_of_ingress=0.093527 dedup_ratio=0.332242 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=7 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.383667009e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87354 payload_rows_remote=76324 dup_rows=43718 dup_rows_remote=38384 dedup_ratio=0.333542 router_tokens_per_expert=[455, 483, 541, 515, 535, 481, 532, 518, 502, 496, 495, 521, 552, 535, 516, 503, 513, 558, 518, 535, 477, 566, 512, 517, 518, 487, 504, 516, 497, 514, 476, 500, 521, 504, 497, 499, 511, 511, 556, 494, 449, 473, 492, 525, 486, 484, 525, 540, 556, 481, 501, 498, 502, 498, 522, 488, 489, 520, 534, 470, 506, 524, 497, 507, 491, 521, 499, 505, 483, 532, 538, 528, 519, 536, 533, 516, 509, 518, 540, 482, 536, 529, 501, 506, 504, 511, 517, 499, 527, 561, 534, 562, 503, 499, 511, 507, 507, 475, 499, 530, 565, 544, 483, 476, 538, 550, 464, 484, 533, 523, 485, 526, 517, 502, 477, 479, 472, 512, 608, 550, 500, 529, 519, 494, 496, 521, 547, 526] payload_rows_per_dst=[10928, 11030, 10798, 10794, 10884, 10960, 11030, 10930] dup_rows_per_dst=[5432, 5386, 5336, 5392, 5616, 5654, 5334, 5568] combine_rows_per_src=[10928, 11030, 10798, 10794, 10884, 10960, 11030, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87624 payload_rows_remote=76634 dup_rows=43448 dup_rows_remote=37944 dedup_ratio=0.331482 router_tokens_per_expert=[530, 525, 508, 508, 535, 532, 515, 512, 437, 488, 555, 531, 525, 561, 515, 539, 558, 523, 526, 476, 540, 538, 505, 546, 486, 510, 500, 527, 520, 500, 501, 501, 523, 511, 508, 510, 487, 500, 532, 495, 528, 507, 528, 507, 515, 516, 533, 547, 494, 511, 512, 524, 551, 541, 497, 514, 496, 480, 484, 546, 471, 479, 510, 508, 503, 516, 549, 503, 453, 491, 525, 483, 522, 516, 545, 516, 521, 514, 522, 509, 538, 490, 492, 472, 499, 499, 501, 518, 492, 510, 520, 514, 547, 490, 517, 531, 512, 529, 477, 521, 487, 472, 523, 547, 477, 495, 518, 501, 503, 508, 498, 549, 540, 497, 524, 515, 514, 525, 510, 520, 508, 495, 484, 544, 500, 480, 485, 522] payload_rows_per_dst=[11038, 11022, 10990, 10884, 10844, 10940, 10906, 11000] dup_rows_per_dst=[5594, 5492, 5504, 5352, 5532, 5320, 5328, 5326] combine_rows_per_src=[11038, 11022, 10990, 10884, 10844, 10940, 10906, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76666 dup_rows=43528 dup_rows_remote=38214 dedup_ratio=0.332092 router_tokens_per_expert=[508, 519, 474, 523, 505, 502, 538, 505, 527, 521, 478, 498, 500, 504, 496, 478, 521, 486, 536, 522, 522, 527, 537, 512, 540, 507, 516, 538, 510, 523, 497, 488, 506, 512, 467, 461, 536, 541, 523, 507, 505, 515, 479, 537, 544, 512, 508, 542, 476, 503, 522, 514, 505, 528, 480, 529, 469, 534, 548, 488, 532, 508, 526, 505, 514, 529, 536, 520, 491, 515, 515, 489, 486, 481, 549, 496, 525, 508, 517, 545, 512, 513, 536, 539, 508, 465, 464, 493, 485, 506, 489, 521, 520, 503, 517, 503, 525, 506, 528, 525, 516, 487, 500, 536, 526, 491, 524, 522, 517, 514, 502, 508, 532, 529, 550, 540, 522, 532, 528, 493, 535, 505, 510, 521, 497, 516, 485, 504] payload_rows_per_dst=[10878, 11206, 11074, 10910, 10864, 10782, 10996, 10834] dup_rows_per_dst=[5314, 5788, 5576, 5424, 5308, 5366, 5418, 5334] combine_rows_per_src=[10878, 11206, 11074, 10910, 10864, 10782, 10996, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87514 payload_rows_remote=76566 dup_rows=43558 dup_rows_remote=38082 dedup_ratio=0.332321 router_tokens_per_expert=[481, 532, 531, 494, 512, 529, 493, 493, 531, 523, 529, 516, 538, 533, 523, 534, 516, 484, 519, 495, 506, 480, 497, 495, 507, 515, 518, 506, 468, 516, 528, 486, 497, 523, 537, 537, 499, 499, 545, 544, 506, 517, 514, 555, 501, 523, 512, 515, 520, 523, 519, 504, 496, 523, 537, 503, 494, 544, 535, 532, 477, 485, 533, 487, 575, 504, 506, 549, 530, 527, 510, 499, 505, 501, 526, 501, 493, 467, 486, 497, 483, 497, 530, 501, 543, 466, 511, 561, 552, 516, 510, 503, 537, 517, 542, 494, 500, 547, 526, 550, 472, 499, 527, 521, 506, 455, 490, 525, 522, 504, 499, 452, 518, 523, 481, 510, 479, 489, 512, 512, 507, 522, 488, 528, 534, 489, 524, 522] payload_rows_per_dst=[11046, 10826, 11110, 10948, 10902, 10956, 10888, 10838] dup_rows_per_dst=[5538, 5246, 5538, 5476, 5450, 5570, 5302, 5438] combine_rows_per_src=[11046, 10826, 11110, 10948, 10902, 10956, 10888, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625623040 dispatch_meta_bytes=458928 combine_payload_bytes=625623040 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626081968 ingress_bytes=701120512 link_bytes=701120512 token_bytes=1251705008 weight_movement_bytes=75497472 total_nvlink_bytes=1327202480 weight_to_token_ratio=0.060316 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.891194994e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627589120 dispatch_meta_bytes=459384 combine_payload_bytes=627589120 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628048504 ingress_bytes=854081536 link_bytes=854081536 token_bytes=1255637624 weight_movement_bytes=226492416 total_nvlink_bytes=1482130040 weight_to_token_ratio=0.180380 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083095609e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625246208 dispatch_meta_bytes=458832 combine_payload_bytes=625246208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625705040 ingress_bytes=700743680 link_bytes=700743680 token_bytes=1250951248 weight_movement_bytes=75497472 total_nvlink_bytes=1326448720 weight_to_token_ratio=0.060352 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.886416234e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=458312 combine_payload_bytes=627785728 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628244040 ingress_bytes=778780672 link_bytes=778780672 token_bytes=1256029768 weight_movement_bytes=150994944 total_nvlink_bytes=1407024712 weight_to_token_ratio=0.120216 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876035138e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628047872 dispatch_meta_bytes=459520 combine_payload_bytes=628047872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628507392 ingress_bytes=703545344 link_bytes=703545344 token_bytes=1256555264 weight_movement_bytes=75497472 total_nvlink_bytes=1332052736 weight_to_token_ratio=0.060083 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921945277e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627228672 dispatch_meta_bytes=458592 combine_payload_bytes=627228672 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627687264 ingress_bytes=702726144 link_bytes=702726144 token_bytes=1254915936 weight_movement_bytes=75497472 total_nvlink_bytes=1330413408 weight_to_token_ratio=0.060161 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.911556668e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15055848104 prefetch_bytes=830472192 reduce_bytes=754974720 weight_movement_bytes=1585446912 egress_bytes=7530676904 ingress_bytes=9110618112 link_bytes=9110618112 total_nvlink_bytes=16641295016 weight_to_token_ratio=0.105304 prefetch_share_of_ingress=0.091154 dedup_ratio=0.332605 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=10 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041820 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.627955152e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15046933952 prefetch_bytes=830472192 reduce_bytes=679477248 weight_movement_bytes=1509949440 egress_bytes=7526219200 ingress_bytes=9030664192 link_bytes=9030664192 total_nvlink_bytes=16556883392 weight_to_token_ratio=0.100349 prefetch_share_of_ingress=0.091961 dedup_ratio=0.332755 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=9 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.543461131e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15060336376 prefetch_bytes=679477248 reduce_bytes=905969664 weight_movement_bytes=1585446912 egress_bytes=7532920568 ingress_bytes=9112862720 link_bytes=9112862720 total_nvlink_bytes=16645783288 weight_to_token_ratio=0.105273 prefetch_share_of_ingress=0.074562 dedup_ratio=0.332113 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=12 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043811 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.630327218e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15065256360 prefetch_bytes=830472192 reduce_bytes=452984832 weight_movement_bytes=1283457024 egress_bytes=7535382952 ingress_bytes=8813330432 link_bytes=8813330432 total_nvlink_bytes=16348713384 weight_to_token_ratio=0.085193 prefetch_share_of_ingress=0.094229 dedup_ratio=0.332372 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=6 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.313786298e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15069737488 prefetch_bytes=679477248 reduce_bytes=1132462080 weight_movement_bytes=1811939328 egress_bytes=7537619472 ingress_bytes=9344057344 link_bytes=9344057344 total_nvlink_bytes=16881676816 weight_to_token_ratio=0.120237 prefetch_share_of_ingress=0.072718 dedup_ratio=0.331618 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=15 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.874649990e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87500 payload_rows_remote=76582 dup_rows=43572 dup_rows_remote=38240 dedup_ratio=0.332428 router_tokens_per_expert=[484, 514, 525, 527, 494, 475, 496, 503, 529, 494, 475, 507, 468, 549, 505, 512, 531, 512, 525, 526, 500, 529, 541, 503, 539, 533, 519, 497, 543, 513, 499, 501, 495, 523, 510, 518, 524, 493, 508, 493, 492, 565, 498, 526, 530, 505, 493, 493, 497, 540, 521, 522, 495, 492, 532, 506, 546, 510, 536, 495, 526, 495, 502, 481, 502, 533, 491, 498, 500, 546, 504, 508, 513, 504, 520, 525, 471, 501, 480, 491, 532, 515, 533, 538, 479, 510, 501, 485, 520, 504, 533, 528, 472, 517, 522, 521, 553, 514, 521, 523, 509, 494, 502, 484, 500, 513, 494, 540, 500, 501, 482, 485, 558, 516, 564, 551, 497, 527, 524, 497, 510, 524, 587, 479, 525, 523, 523, 489] payload_rows_per_dst=[10918, 10996, 10926, 10878, 10818, 11084, 10924, 10956] dup_rows_per_dst=[5332, 5632, 5400, 5514, 5356, 5488, 5306, 5544] combine_rows_per_src=[10918, 10996, 10926, 10878, 10818, 11084, 10924, 10956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=8 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627359744 dispatch_meta_bytes=459288 combine_payload_bytes=627359744 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627819032 ingress_bytes=778354688 link_bytes=778354688 token_bytes=1255178776 weight_movement_bytes=150994944 total_nvlink_bytes=1406173720 weight_to_token_ratio=0.120298 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.870633061e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=8 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15050074136 prefetch_bytes=754974720 reduce_bytes=981467136 weight_movement_bytes=1736441856 egress_bytes=7527786520 ingress_bytes=9258729472 link_bytes=9258729472 total_nvlink_bytes=16786515992 weight_to_token_ratio=0.115378 prefetch_share_of_ingress=0.081542 dedup_ratio=0.332462 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=13 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.784476863e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none + [2026-08-29 23:36:24] iteration 9/ 12 | consumed samples: 72 | elapsed time per iteration (ms): 357.2 | throughput per GPU (TFLOP/s/GPU): 506.2 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.528523E+00 | loss scale: 1.0 | grad norm: 5.548 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87316 payload_rows_remote=76496 dup_rows=43756 dup_rows_remote=38400 dedup_ratio=0.333832 router_tokens_per_expert=[494, 511, 535, 507, 524, 522, 501, 517, 482, 498, 522, 508, 534, 522, 470, 527, 533, 493, 534, 521, 495, 487, 519, 526, 491, 525, 484, 480, 521, 484, 520, 475, 481, 491, 505, 537, 514, 536, 492, 529, 512, 518, 488, 507, 517, 490, 533, 511, 512, 515, 535, 531, 522, 510, 535, 546, 540, 531, 520, 507, 533, 496, 539, 526, 525, 479, 524, 486, 520, 508, 508, 517, 520, 494, 480, 502, 567, 529, 491, 522, 523, 495, 514, 506, 521, 492, 494, 512, 510, 478, 488, 487, 543, 512, 519, 516, 523, 540, 502, 512, 514, 531, 534, 486, 551, 503, 498, 515, 522, 535, 484, 492, 467, 484, 536, 491, 522, 508, 519, 529, 475, 526, 521, 534, 501, 559, 513, 506] payload_rows_per_dst=[10858, 10820, 10902, 11118, 10860, 10814, 11022, 10922] dup_rows_per_dst=[5490, 5356, 5420, 5678, 5484, 5406, 5462, 5460] combine_rows_per_src=[10858, 10820, 10902, 11118, 10860, 10814, 11022, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76476 dup_rows=43584 dup_rows_remote=37966 dedup_ratio=0.332520 router_tokens_per_expert=[526, 508, 509, 510, 549, 514, 504, 490, 520, 519, 555, 497, 444, 534, 515, 515, 535, 499, 509, 464, 512, 528, 473, 514, 489, 519, 495, 509, 493, 531, 533, 506, 471, 515, 503, 526, 537, 548, 544, 518, 519, 546, 533, 519, 511, 495, 521, 509, 511, 483, 519, 503, 530, 473, 489, 522, 531, 482, 494, 516, 510, 533, 511, 525, 508, 529, 571, 513, 508, 518, 502, 539, 540, 525, 479, 547, 531, 518, 537, 531, 514, 522, 495, 514, 493, 524, 534, 505, 514, 509, 518, 474, 491, 485, 512, 509, 487, 465, 510, 488, 504, 472, 527, 546, 495, 526, 512, 496, 541, 534, 492, 499, 496, 528, 513, 532, 493, 507, 525, 513, 495, 522, 492, 503, 552, 498, 473, 526] payload_rows_per_dst=[10906, 10922, 11012, 10878, 11134, 10870, 10848, 10918] dup_rows_per_dst=[5512, 5296, 5618, 5386, 5658, 5356, 5340, 5418] combine_rows_per_src=[10906, 10922, 11012, 10878, 11134, 10870, 10848, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76632 dup_rows=43454 dup_rows_remote=38056 dedup_ratio=0.331528 router_tokens_per_expert=[508, 515, 528, 495, 518, 513, 496, 519, 527, 552, 513, 555, 516, 560, 487, 550, 526, 476, 504, 549, 531, 494, 519, 527, 508, 533, 488, 481, 511, 503, 523, 506, 517, 543, 473, 513, 516, 498, 513, 528, 546, 553, 498, 493, 499, 510, 533, 516, 528, 479, 505, 521, 443, 473, 512, 507, 539, 498, 493, 480, 498, 511, 510, 463, 514, 518, 492, 479, 509, 462, 481, 497, 512, 516, 509, 535, 526, 511, 499, 535, 558, 532, 489, 505, 542, 533, 495, 473, 524, 519, 502, 524, 538, 572, 520, 482, 486, 507, 486, 520, 549, 526, 508, 502, 504, 517, 505, 514, 518, 509, 544, 497, 521, 539, 518, 524, 504, 554, 554, 547, 457, 498, 496, 524, 486, 509, 478, 492] payload_rows_per_dst=[11002, 10972, 10970, 10774, 10860, 11134, 10986, 10920] dup_rows_per_dst=[5702, 5386, 5528, 5146, 5330, 5482, 5398, 5482] combine_rows_per_src=[11002, 10972, 10970, 10774, 10860, 11134, 10986, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76464 dup_rows=43534 dup_rows_remote=37878 dedup_ratio=0.332138 router_tokens_per_expert=[478, 532, 502, 534, 446, 495, 540, 515, 498, 519, 526, 490, 515, 536, 469, 519, 478, 492, 526, 513, 527, 510, 521, 521, 517, 526, 483, 525, 531, 481, 526, 500, 495, 478, 503, 542, 491, 507, 497, 508, 510, 541, 490, 524, 504, 536, 545, 488, 511, 533, 534, 531, 468, 497, 488, 544, 545, 541, 486, 541, 531, 509, 500, 510, 491, 527, 576, 534, 468, 544, 513, 514, 545, 436, 493, 497, 525, 491, 532, 510, 505, 481, 537, 517, 501, 511, 481, 513, 513, 513, 476, 518, 492, 521, 518, 491, 512, 497, 506, 493, 522, 534, 499, 522, 513, 540, 502, 503, 488, 553, 518, 471, 500, 539, 504, 508, 594, 540, 503, 543, 500, 520, 537, 513, 525, 531, 498, 505] payload_rows_per_dst=[10864, 10966, 10946, 10712, 10938, 11066, 10972, 11074] dup_rows_per_dst=[5364, 5388, 5372, 5372, 5430, 5564, 5388, 5656] combine_rows_per_src=[10864, 10966, 10946, 10712, 10938, 11066, 10972, 11074] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76638 dup_rows=43492 dup_rows_remote=38018 dedup_ratio=0.331818 router_tokens_per_expert=[509, 500, 531, 515, 500, 521, 521, 521, 486, 478, 514, 481, 508, 509, 527, 516, 502, 497, 520, 511, 520, 510, 496, 524, 527, 522, 551, 489, 513, 510, 509, 526, 498, 502, 504, 520, 506, 533, 524, 501, 515, 531, 552, 522, 499, 510, 470, 514, 519, 506, 485, 475, 515, 538, 510, 505, 512, 507, 496, 520, 517, 520, 528, 555, 500, 506, 505, 492, 505, 493, 538, 525, 497, 539, 508, 485, 508, 520, 480, 529, 494, 480, 484, 543, 513, 530, 505, 529, 507, 511, 507, 531, 509, 548, 477, 521, 537, 512, 499, 513, 498, 481, 508, 504, 538, 512, 495, 544, 489, 472, 513, 518, 544, 505, 532, 530, 536, 482, 511, 512, 524, 510, 536, 513, 514, 522, 525, 515] payload_rows_per_dst=[10810, 10966, 10906, 10942, 10928, 11036, 10910, 11082] dup_rows_per_dst=[5464, 5488, 5496, 5474, 5332, 5342, 5356, 5540] combine_rows_per_src=[10810, 10966, 10906, 10942, 10928, 11036, 10910, 11082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87312 payload_rows_remote=76492 dup_rows=43760 dup_rows_remote=38366 dedup_ratio=0.333862 router_tokens_per_expert=[559, 494, 540, 541, 538, 461, 477, 524, 511, 516, 507, 491, 475, 514, 547, 551, 555, 503, 469, 532, 526, 515, 536, 481, 465, 526, 510, 531, 547, 522, 504, 527, 521, 508, 505, 516, 487, 542, 513, 527, 516, 482, 542, 552, 472, 516, 512, 468, 509, 550, 511, 539, 497, 498, 515, 513, 533, 513, 530, 485, 540, 491, 518, 526, 491, 512, 525, 519, 514, 527, 512, 493, 498, 510, 484, 504, 522, 513, 511, 486, 520, 506, 515, 487, 482, 499, 540, 518, 506, 508, 475, 511, 490, 522, 493, 535, 524, 547, 527, 497, 556, 496, 526, 480, 499, 529, 482, 500, 514, 564, 492, 483, 514, 513, 523, 511, 478, 519, 532, 492, 524, 495, 505, 529, 473, 506, 506, 530] payload_rows_per_dst=[10910, 10946, 10976, 10910, 10786, 10820, 10996, 10968] dup_rows_per_dst=[5582, 5552, 5382, 5626, 5456, 5394, 5436, 5332] combine_rows_per_src=[10910, 10946, 10976, 10910, 10786, 10820, 10996, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87276 payload_rows_remote=76300 dup_rows=43796 dup_rows_remote=38316 dedup_ratio=0.334137 router_tokens_per_expert=[514, 542, 481, 532, 502, 507, 504, 521, 481, 499, 543, 461, 504, 526, 543, 500, 494, 484, 491, 510, 514, 509, 514, 480, 567, 506, 546, 519, 554, 454, 557, 498, 544, 501, 525, 499, 502, 502, 534, 497, 515, 479, 523, 506, 508, 510, 487, 480, 505, 514, 521, 503, 545, 513, 528, 460, 558, 544, 521, 465, 526, 482, 507, 499, 486, 521, 484, 504, 553, 521, 491, 534, 517, 531, 504, 524, 505, 491, 513, 558, 512, 527, 503, 495, 486, 518, 523, 527, 527, 532, 540, 508, 498, 541, 495, 504, 501, 518, 554, 514, 543, 507, 495, 557, 557, 526, 525, 509, 531, 499, 474, 488, 495, 525, 490, 496, 527, 482, 505, 531, 540, 504, 486, 501, 476, 516, 490, 541] payload_rows_per_dst=[10976, 10932, 10932, 10934, 10778, 10982, 10984, 10758] dup_rows_per_dst=[5480, 5662, 5366, 5500, 5422, 5490, 5612, 5264] combine_rows_per_src=[10976, 10932, 10932, 10934, 10778, 10982, 10984, 10758] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626655232 dispatch_meta_bytes=459584 combine_payload_bytes=626655232 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627114816 ingress_bytes=702152704 link_bytes=702152704 token_bytes=1253770048 weight_movement_bytes=75497472 total_nvlink_bytes=1329267520 weight_to_token_ratio=0.060216 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.904284641e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=458752 combine_payload_bytes=627769344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628228096 ingress_bytes=703266816 link_bytes=703266816 token_bytes=1255997440 weight_movement_bytes=75497472 total_nvlink_bytes=1331494912 weight_to_token_ratio=0.060110 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918413150e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626491392 dispatch_meta_bytes=457768 combine_payload_bytes=626491392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626949160 ingress_bytes=701988864 link_bytes=701988864 token_bytes=1253440552 weight_movement_bytes=75497472 total_nvlink_bytes=1328938024 weight_to_token_ratio=0.060232 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.902206920e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626393088 dispatch_meta_bytes=457368 combine_payload_bytes=626393088 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626850456 ingress_bytes=852885504 link_bytes=852885504 token_bytes=1253243544 weight_movement_bytes=226492416 total_nvlink_bytes=1479735960 weight_to_token_ratio=0.180725 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081578872e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627818496 dispatch_meta_bytes=458624 combine_payload_bytes=627818496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628277120 ingress_bytes=778813440 link_bytes=778813440 token_bytes=1256095616 weight_movement_bytes=150994944 total_nvlink_bytes=1407090560 weight_to_token_ratio=0.120210 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876450682e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87370 payload_rows_remote=76306 dup_rows=43702 dup_rows_remote=38090 dedup_ratio=0.333420 router_tokens_per_expert=[503, 530, 516, 508, 489, 485, 498, 499, 493, 536, 529, 489, 540, 458, 512, 491, 520, 528, 512, 484, 567, 527, 445, 520, 504, 543, 467, 513, 532, 502, 508, 538, 497, 526, 481, 489, 500, 502, 528, 490, 508, 530, 510, 543, 496, 490, 521, 512, 528, 525, 529, 465, 510, 562, 514, 523, 501, 527, 551, 498, 527, 496, 505, 550, 526, 502, 484, 531, 537, 521, 493, 515, 512, 515, 526, 515, 484, 572, 553, 552, 505, 521, 523, 507, 521, 482, 512, 539, 514, 499, 488, 536, 517, 512, 514, 468, 525, 475, 521, 523, 493, 476, 544, 489, 511, 529, 503, 511, 490, 504, 543, 544, 498, 535, 487, 509, 506, 531, 509, 498, 496, 482, 513, 511, 551, 495, 520, 498] payload_rows_per_dst=[10914, 11008, 10798, 10992, 11064, 10868, 10878, 10848] dup_rows_per_dst=[5238, 5412, 5448, 5630, 5612, 5448, 5484, 5430] combine_rows_per_src=[10914, 11008, 10798, 10992, 11064, 10868, 10878, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626622464 dispatch_meta_bytes=459432 combine_payload_bytes=626622464 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627081896 ingress_bytes=702119936 link_bytes=702119936 token_bytes=1253704360 weight_movement_bytes=75497472 total_nvlink_bytes=1329201832 weight_to_token_ratio=0.060220 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903869097e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625049600 dispatch_meta_bytes=458464 combine_payload_bytes=625049600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625508064 ingress_bytes=700547072 link_bytes=700547072 token_bytes=1250557664 weight_movement_bytes=75497472 total_nvlink_bytes=1326055136 weight_to_token_ratio=0.060371 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.883922968e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625098752 dispatch_meta_bytes=457584 combine_payload_bytes=625098752 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=625556336 ingress_bytes=927088640 link_bytes=927088640 token_bytes=1250655088 weight_movement_bytes=301989888 total_nvlink_bytes=1552644976 weight_to_token_ratio=0.241465 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.175678893e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87184 payload_rows_remote=76234 dup_rows=43888 dup_rows_remote=38222 dedup_ratio=0.334839 router_tokens_per_expert=[523, 533, 535, 497, 469, 476, 507, 507, 501, 536, 535, 516, 556, 490, 498, 481, 532, 565, 496, 532, 537, 500, 559, 510, 555, 506, 522, 481, 533, 493, 509, 478, 533, 485, 534, 535, 489, 508, 543, 518, 464, 478, 460, 446, 551, 462, 501, 478, 546, 522, 534, 515, 535, 554, 564, 470, 550, 542, 512, 460, 466, 514, 522, 518, 550, 518, 519, 505, 555, 487, 498, 485, 512, 534, 524, 476, 521, 482, 522, 481, 508, 542, 466, 477, 493, 527, 522, 524, 557, 474, 492, 507, 496, 514, 513, 494, 494, 494, 516, 541, 505, 505, 468, 521, 520, 528, 578, 511, 500, 529, 525, 547, 529, 502, 542, 506, 533, 486, 521, 528, 498, 517, 532, 500, 507, 488, 520, 493] payload_rows_per_dst=[10916, 10950, 10668, 11006, 10890, 10920, 10958, 10876] dup_rows_per_dst=[5404, 5666, 5302, 5642, 5448, 5292, 5606, 5528] combine_rows_per_src=[10916, 10950, 10668, 11006, 10890, 10920, 10958, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87246 payload_rows_remote=76312 dup_rows=43826 dup_rows_remote=38426 dedup_ratio=0.334366 router_tokens_per_expert=[473, 568, 509, 500, 530, 492, 537, 545, 530, 494, 534, 544, 521, 521, 542, 538, 511, 518, 520, 521, 514, 496, 535, 488, 498, 500, 514, 527, 513, 521, 514, 494, 521, 545, 503, 541, 490, 500, 518, 511, 544, 480, 516, 493, 541, 519, 506, 515, 500, 492, 515, 487, 524, 474, 544, 506, 508, 500, 504, 525, 487, 475, 520, 483, 527, 525, 496, 533, 557, 486, 514, 522, 503, 500, 497, 531, 518, 515, 488, 519, 535, 535, 492, 536, 542, 501, 493, 497, 518, 517, 518, 513, 518, 490, 482, 495, 506, 513, 519, 495, 503, 518, 551, 517, 503, 501, 504, 498, 482, 539, 497, 521, 518, 513, 507, 502, 506, 506, 490, 493, 544, 498, 505, 499, 492, 502, 522, 510] payload_rows_per_dst=[11004, 10930, 11018, 10710, 10946, 10818, 10934, 10886] dup_rows_per_dst=[5752, 5438, 5468, 5378, 5516, 5546, 5400, 5328] combine_rows_per_src=[11004, 10930, 11018, 10710, 10946, 10818, 10934, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87576 payload_rows_remote=76682 dup_rows=43496 dup_rows_remote=38114 dedup_ratio=0.331848 router_tokens_per_expert=[538, 516, 493, 530, 494, 531, 500, 531, 541, 509, 488, 521, 496, 482, 513, 511, 529, 505, 537, 505, 502, 478, 544, 509, 508, 538, 478, 541, 543, 530, 517, 499, 494, 521, 469, 491, 480, 522, 515, 509, 517, 506, 520, 510, 532, 529, 491, 532, 510, 524, 533, 465, 518, 490, 531, 528, 534, 518, 491, 529, 524, 515, 524, 512, 531, 514, 512, 490, 489, 495, 489, 521, 489, 511, 516, 567, 506, 495, 494, 495, 507, 509, 519, 520, 547, 494, 484, 476, 507, 500, 512, 518, 494, 524, 477, 516, 504, 513, 476, 547, 517, 524, 519, 437, 523, 514, 521, 514, 470, 486, 543, 527, 486, 548, 520, 521, 518, 490, 558, 518, 489, 536, 530, 506, 538, 512, 536, 536] payload_rows_per_dst=[10944, 11042, 10894, 10876, 10848, 10926, 10952, 11094] dup_rows_per_dst=[5444, 5484, 5382, 5616, 5380, 5282, 5318, 5590] combine_rows_per_src=[10944, 11042, 10894, 10876, 10848, 10926, 10952, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87600 payload_rows_remote=76528 dup_rows=43472 dup_rows_remote=37972 dedup_ratio=0.331665 router_tokens_per_expert=[513, 548, 523, 495, 505, 524, 497, 503, 548, 496, 510, 488, 535, 476, 530, 523, 473, 529, 494, 489, 507, 524, 495, 549, 522, 495, 519, 544, 499, 525, 470, 511, 545, 507, 502, 493, 505, 502, 523, 488, 516, 487, 508, 518, 505, 528, 493, 521, 556, 477, 505, 551, 509, 489, 502, 505, 498, 466, 561, 485, 539, 527, 523, 538, 509, 532, 491, 517, 526, 517, 519, 486, 564, 555, 504, 544, 547, 490, 498, 494, 504, 516, 527, 473, 482, 499, 484, 508, 519, 524, 472, 510, 500, 531, 516, 527, 537, 518, 485, 502, 525, 519, 513, 511, 488, 517, 520, 531, 521, 517, 511, 542, 543, 470, 502, 538, 539, 491, 498, 512, 521, 482, 490, 487, 551, 509, 524, 506] payload_rows_per_dst=[10686, 10762, 11140, 10990, 11050, 10952, 10948, 11072] dup_rows_per_dst=[5032, 5200, 5852, 5472, 5536, 5232, 5648, 5500] combine_rows_per_src=[10686, 10762, 11140, 10990, 11050, 10952, 10948, 11072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87346 payload_rows_remote=76468 dup_rows=43726 dup_rows_remote=38060 dedup_ratio=0.333603 router_tokens_per_expert=[515, 507, 526, 522, 510, 546, 518, 500, 514, 497, 545, 536, 504, 539, 500, 478, 486, 494, 486, 546, 503, 473, 550, 551, 525, 533, 522, 539, 486, 496, 518, 484, 529, 528, 495, 533, 533, 500, 521, 500, 489, 494, 513, 471, 519, 481, 544, 532, 557, 526, 498, 503, 547, 515, 497, 543, 531, 526, 515, 516, 502, 508, 494, 494, 517, 525, 497, 500, 497, 536, 480, 485, 504, 513, 495, 505, 491, 561, 535, 584, 477, 495, 556, 499, 518, 511, 476, 511, 541, 499, 483, 538, 503, 475, 524, 506, 531, 529, 526, 538, 478, 516, 422, 498, 509, 479, 499, 506, 505, 536, 555, 523, 518, 512, 509, 541, 488, 512, 489, 522, 521, 486, 525, 517, 516, 479, 484, 527] payload_rows_per_dst=[10986, 10924, 11036, 10878, 10926, 10864, 10940, 10792] dup_rows_per_dst=[5528, 5460, 5328, 5666, 5524, 5360, 5360, 5500] combine_rows_per_src=[10986, 10924, 11036, 10878, 10926, 10864, 10940, 10792] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87420 payload_rows_remote=76542 dup_rows=43652 dup_rows_remote=38238 dedup_ratio=0.333038 router_tokens_per_expert=[508, 520, 542, 524, 514, 532, 512, 501, 486, 490, 521, 490, 511, 508, 491, 496, 535, 511, 511, 496, 475, 550, 542, 522, 533, 484, 540, 520, 467, 515, 482, 510, 532, 536, 533, 521, 522, 553, 526, 511, 485, 525, 542, 484, 504, 522, 551, 479, 500, 508, 486, 508, 501, 494, 520, 510, 512, 515, 541, 471, 497, 564, 510, 490, 496, 519, 523, 491, 528, 511, 496, 521, 560, 479, 515, 498, 500, 552, 530, 493, 530, 492, 498, 505, 505, 518, 477, 482, 503, 486, 510, 533, 520, 549, 539, 499, 524, 484, 473, 513, 509, 508, 521, 500, 559, 505, 546, 508, 538, 545, 497, 533, 517, 472, 526, 489, 524, 495, 512, 516, 513, 485, 511, 505, 534, 505, 519, 500] payload_rows_per_dst=[10896, 10872, 11024, 10872, 11008, 10878, 11040, 10830] dup_rows_per_dst=[5396, 5514, 5628, 5382, 5416, 5414, 5486, 5416] combine_rows_per_src=[10896, 10872, 11024, 10872, 11008, 10878, 11040, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=624508928 dispatch_meta_bytes=457824 combine_payload_bytes=624508928 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=624966752 ingress_bytes=851001344 link_bytes=851001344 token_bytes=1249475680 weight_movement_bytes=226492416 total_nvlink_bytes=1475968096 weight_to_token_ratio=0.181270 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079189492e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625147904 dispatch_meta_bytes=458952 combine_payload_bytes=625147904 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=625606856 ingress_bytes=851640320 link_bytes=851640320 token_bytes=1250754760 weight_movement_bytes=226492416 total_nvlink_bytes=1477247176 weight_to_token_ratio=0.181085 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079999803e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87448 payload_rows_remote=76458 dup_rows=43624 dup_rows_remote=37990 dedup_ratio=0.332825 router_tokens_per_expert=[492, 517, 524, 508, 532, 533, 522, 486, 497, 504, 536, 529, 515, 501, 509, 509, 497, 485, 536, 529, 491, 522, 492, 519, 487, 509, 528, 508, 530, 511, 517, 491, 520, 512, 502, 472, 509, 588, 516, 504, 511, 520, 469, 491, 493, 518, 506, 468, 486, 470, 506, 507, 500, 552, 513, 465, 521, 523, 488, 514, 538, 506, 501, 530, 513, 511, 534, 462, 513, 498, 560, 502, 520, 475, 561, 510, 511, 532, 554, 509, 500, 497, 497, 509, 556, 514, 483, 493, 532, 520, 548, 508, 490, 532, 571, 544, 529, 480, 514, 532, 509, 499, 533, 564, 517, 526, 518, 557, 496, 549, 503, 503, 467, 522, 497, 507, 498, 514, 472, 489, 549, 485, 519, 522, 502, 475, 530, 515] payload_rows_per_dst=[10990, 10894, 10862, 10842, 10968, 11154, 10848, 10890] dup_rows_per_dst=[5634, 5442, 5336, 5340, 5530, 5650, 5456, 5236] combine_rows_per_src=[10990, 10894, 10862, 10842, 10968, 11154, 10848, 10890] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628178944 dispatch_meta_bytes=459184 combine_payload_bytes=628178944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628638128 ingress_bytes=703676416 link_bytes=703676416 token_bytes=1256817072 weight_movement_bytes=75497472 total_nvlink_bytes=1332314544 weight_to_token_ratio=0.060070 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923607454e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=458000 combine_payload_bytes=626917376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627375376 ingress_bytes=702414848 link_bytes=702414848 token_bytes=1254292752 weight_movement_bytes=75497472 total_nvlink_bytes=1329790224 weight_to_token_ratio=0.060191 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907608996e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626425856 dispatch_meta_bytes=458112 combine_payload_bytes=626425856 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626883968 ingress_bytes=777420800 link_bytes=777420800 token_bytes=1253309824 weight_movement_bytes=150994944 total_nvlink_bytes=1404304768 weight_to_token_ratio=0.120477 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.858790047e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627032064 dispatch_meta_bytes=459120 combine_payload_bytes=627032064 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627491184 ingress_bytes=702529536 link_bytes=702529536 token_bytes=1254523248 weight_movement_bytes=75497472 total_nvlink_bytes=1330020720 weight_to_token_ratio=0.060180 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909063402e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87328 payload_rows_remote=76472 dup_rows=43744 dup_rows_remote=38514 dedup_ratio=0.333740 router_tokens_per_expert=[509, 533, 537, 527, 556, 512, 472, 516, 487, 522, 494, 488, 523, 526, 536, 492, 526, 507, 498, 550, 542, 540, 475, 491, 509, 458, 504, 500, 534, 558, 551, 504, 504, 506, 530, 535, 512, 501, 503, 466, 503, 485, 463, 515, 524, 525, 514, 481, 516, 501, 516, 526, 514, 506, 542, 500, 521, 549, 532, 491, 480, 521, 475, 511, 490, 482, 537, 498, 511, 501, 503, 511, 464, 511, 491, 501, 529, 485, 516, 513, 545, 524, 530, 553, 504, 547, 515, 554, 536, 484, 488, 507, 525, 499, 528, 553, 493, 513, 486, 466, 520, 530, 503, 501, 502, 474, 520, 506, 553, 518, 518, 486, 506, 509, 536, 528, 504, 542, 544, 511, 523, 530, 527, 523, 480, 500, 522, 482] payload_rows_per_dst=[10938, 11004, 10886, 10906, 10856, 11062, 10756, 10920] dup_rows_per_dst=[5522, 5490, 5248, 5496, 5230, 5722, 5422, 5614] combine_rows_per_src=[10938, 11004, 10886, 10906, 10856, 11062, 10756, 10920] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626343936 dispatch_meta_bytes=457792 combine_payload_bytes=626343936 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626801728 ingress_bytes=777338880 link_bytes=777338880 token_bytes=1253145664 weight_movement_bytes=150994944 total_nvlink_bytes=1404140608 weight_to_token_ratio=0.120493 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.857751186e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626458624 dispatch_meta_bytes=459944 combine_payload_bytes=626458624 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=626918568 ingress_bytes=701956096 link_bytes=701956096 token_bytes=1253377192 weight_movement_bytes=75497472 total_nvlink_bytes=1328874664 weight_to_token_ratio=0.060235 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.901791375e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87144 payload_rows_remote=76196 dup_rows=43928 dup_rows_remote=38552 dedup_ratio=0.335144 router_tokens_per_expert=[500, 477, 520, 511, 505, 533, 528, 537, 506, 467, 512, 505, 478, 514, 518, 550, 539, 550, 525, 534, 506, 467, 524, 530, 496, 477, 530, 512, 482, 529, 546, 502, 483, 479, 535, 511, 534, 560, 519, 512, 522, 517, 477, 511, 522, 488, 507, 543, 484, 504, 492, 495, 499, 504, 504, 456, 492, 489, 534, 498, 489, 514, 525, 474, 548, 554, 502, 495, 519, 470, 540, 534, 534, 492, 534, 560, 494, 530, 503, 549, 528, 536, 480, 519, 487, 512, 502, 521, 500, 477, 514, 563, 537, 548, 492, 507, 534, 511, 576, 524, 495, 500, 527, 502, 493, 482, 509, 476, 527, 507, 528, 471, 520, 510, 499, 531, 476, 505, 533, 471, 514, 520, 544, 534, 505, 497, 534, 517] payload_rows_per_dst=[10874, 10994, 10954, 10692, 11060, 10788, 10948, 10834] dup_rows_per_dst=[5448, 5504, 5486, 5214, 5656, 5658, 5376, 5586] combine_rows_per_src=[10874, 10994, 10954, 10692, 11060, 10788, 10948, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87398 payload_rows_remote=76386 dup_rows=43674 dup_rows_remote=38346 dedup_ratio=0.333206 router_tokens_per_expert=[497, 498, 521, 534, 515, 554, 498, 520, 522, 488, 529, 541, 525, 516, 530, 499, 501, 473, 520, 509, 520, 494, 485, 523, 538, 515, 511, 488, 513, 540, 535, 505, 532, 491, 513, 500, 519, 537, 541, 500, 503, 548, 556, 501, 529, 518, 498, 523, 507, 474, 495, 516, 496, 506, 519, 493, 518, 515, 490, 465, 500, 497, 537, 539, 509, 483, 498, 517, 512, 543, 511, 491, 477, 497, 512, 518, 541, 490, 543, 470, 466, 470, 538, 556, 525, 534, 491, 551, 538, 522, 504, 515, 515, 539, 483, 499, 538, 520, 477, 520, 498, 486, 516, 514, 523, 547, 493, 475, 500, 516, 476, 528, 531, 496, 503, 510, 477, 524, 549, 525, 503, 543, 514, 516, 500, 520, 516, 491] payload_rows_per_dst=[10982, 11012, 10998, 10840, 10850, 10974, 10824, 10918] dup_rows_per_dst=[5592, 5328, 5620, 5294, 5374, 5518, 5430, 5518] combine_rows_per_src=[10982, 11012, 10998, 10840, 10850, 10974, 10824, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87586 payload_rows_remote=76782 dup_rows=43486 dup_rows_remote=38194 dedup_ratio=0.331772 router_tokens_per_expert=[548, 515, 481, 528, 518, 548, 490, 522, 468, 495, 484, 531, 492, 538, 508, 494, 486, 491, 498, 476, 523, 550, 503, 523, 525, 524, 515, 500, 524, 514, 515, 526, 523, 485, 494, 477, 518, 478, 545, 489, 511, 512, 505, 463, 520, 469, 514, 545, 539, 518, 510, 524, 480, 538, 520, 503, 518, 513, 547, 491, 539, 522, 524, 485, 479, 542, 509, 468, 516, 506, 507, 498, 527, 494, 547, 517, 500, 530, 477, 495, 534, 500, 525, 501, 488, 513, 510, 510, 539, 527, 492, 523, 542, 514, 522, 517, 485, 501, 512, 537, 519, 484, 549, 497, 517, 537, 490, 481, 526, 524, 502, 500, 536, 551, 537, 504, 499, 540, 512, 527, 514, 547, 513, 504, 506, 527, 502, 515] payload_rows_per_dst=[10860, 10934, 10804, 11030, 10862, 11008, 11020, 11068] dup_rows_per_dst=[5460, 5452, 5292, 5512, 5362, 5506, 5302, 5600] combine_rows_per_src=[10860, 10934, 10804, 11030, 10862, 11008, 11020, 11068] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87426 payload_rows_remote=76388 dup_rows=43646 dup_rows_remote=38246 dedup_ratio=0.332993 router_tokens_per_expert=[484, 533, 537, 547, 499, 497, 547, 503, 489, 522, 487, 514, 467, 533, 500, 493, 546, 516, 529, 474, 537, 527, 497, 517, 508, 505, 496, 530, 476, 514, 498, 502, 508, 497, 513, 521, 524, 490, 531, 547, 515, 512, 508, 492, 505, 503, 485, 530, 560, 527, 521, 529, 495, 520, 499, 491, 529, 521, 530, 514, 525, 514, 512, 483, 517, 522, 521, 536, 483, 488, 478, 540, 530, 504, 499, 501, 489, 492, 496, 511, 503, 485, 517, 515, 540, 514, 519, 523, 529, 537, 512, 513, 538, 511, 505, 488, 516, 506, 503, 549, 519, 505, 507, 521, 483, 526, 525, 544, 523, 459, 502, 536, 564, 534, 488, 511, 504, 498, 487, 510, 492, 520, 544, 499, 493, 483, 515, 539] payload_rows_per_dst=[10912, 10788, 10908, 11038, 10888, 10898, 10956, 11038] dup_rows_per_dst=[5392, 5544, 5466, 5502, 5326, 5524, 5492, 5400] combine_rows_per_src=[10912, 10788, 10908, 11038, 10888, 10898, 10956, 11038] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76616 dup_rows=43492 dup_rows_remote=38020 dedup_ratio=0.331818 router_tokens_per_expert=[501, 488, 560, 519, 552, 533, 494, 523, 484, 513, 511, 546, 489, 536, 496, 541, 512, 540, 538, 505, 540, 505, 496, 491, 502, 483, 519, 523, 505, 495, 494, 533, 538, 472, 498, 513, 477, 529, 501, 523, 477, 490, 530, 484, 521, 505, 538, 532, 515, 500, 483, 512, 508, 533, 509, 516, 511, 505, 516, 524, 521, 512, 548, 505, 555, 503, 537, 511, 531, 495, 540, 538, 565, 491, 475, 484, 557, 507, 504, 506, 493, 491, 541, 478, 503, 474, 498, 471, 518, 494, 512, 538, 508, 527, 494, 510, 539, 539, 514, 493, 514, 460, 547, 529, 523, 495, 493, 496, 542, 503, 506, 478, 495, 515, 483, 506, 514, 519, 519, 504, 500, 503, 519, 535, 530, 506, 520, 535] payload_rows_per_dst=[10988, 10936, 10934, 10964, 11104, 10820, 10956, 10878] dup_rows_per_dst=[5584, 5426, 5322, 5472, 5494, 5280, 5386, 5528] combine_rows_per_src=[10988, 10936, 10934, 10964, 11104, 10820, 10956, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87708 payload_rows_remote=76756 dup_rows=43364 dup_rows_remote=37864 dedup_ratio=0.330841 router_tokens_per_expert=[484, 528, 513, 504, 550, 521, 486, 491, 528, 537, 555, 460, 500, 533, 468, 500, 512, 487, 498, 524, 503, 515, 504, 503, 542, 505, 527, 513, 514, 539, 472, 505, 522, 511, 497, 519, 501, 516, 496, 532, 523, 496, 522, 514, 500, 552, 490, 514, 498, 535, 511, 484, 475, 563, 521, 513, 510, 527, 536, 540, 487, 496, 511, 513, 530, 479, 538, 527, 519, 506, 514, 578, 550, 519, 480, 502, 563, 540, 493, 492, 547, 506, 540, 486, 511, 531, 492, 520, 533, 507, 462, 523, 495, 506, 496, 571, 499, 460, 533, 488, 475, 505, 503, 494, 485, 505, 508, 535, 501, 519, 477, 506, 518, 560, 473, 495, 497, 520, 539, 520, 532, 504, 517, 494, 504, 528, 517, 523] payload_rows_per_dst=[10958, 11084, 10930, 10854, 11178, 10952, 10666, 11086] dup_rows_per_dst=[5358, 5242, 5480, 5586, 5482, 5500, 5320, 5396] combine_rows_per_src=[10958, 11084, 10930, 10854, 11178, 10952, 10666, 11086] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=624197632 dispatch_meta_bytes=458992 combine_payload_bytes=624197632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624656624 ingress_bytes=699695104 link_bytes=699695104 token_bytes=1248854256 weight_movement_bytes=75497472 total_nvlink_bytes=1324351728 weight_to_token_ratio=0.060453 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.873118814e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625754112 dispatch_meta_bytes=458928 combine_payload_bytes=625754112 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626213040 ingress_bytes=776749056 link_bytes=776749056 token_bytes=1251967152 weight_movement_bytes=150994944 total_nvlink_bytes=1402962096 weight_to_token_ratio=0.120606 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850271387e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628998144 dispatch_meta_bytes=459904 combine_payload_bytes=628998144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629458048 ingress_bytes=704495616 link_bytes=704495616 token_bytes=1258456192 weight_movement_bytes=75497472 total_nvlink_bytes=1333953664 weight_to_token_ratio=0.059992 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.933996063e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87466 payload_rows_remote=76528 dup_rows=43606 dup_rows_remote=38214 dedup_ratio=0.332687 router_tokens_per_expert=[499, 523, 544, 476, 523, 483, 502, 510, 462, 504, 473, 528, 557, 523, 535, 513, 512, 519, 519, 504, 501, 531, 542, 563, 512, 489, 482, 515, 521, 510, 535, 561, 546, 516, 505, 521, 518, 505, 483, 473, 476, 507, 551, 501, 530, 510, 508, 509, 541, 477, 512, 522, 525, 478, 511, 501, 496, 516, 512, 481, 488, 501, 472, 505, 508, 526, 543, 495, 550, 520, 490, 477, 523, 537, 491, 521, 504, 476, 530, 515, 527, 530, 500, 495, 570, 522, 488, 519, 519, 550, 524, 514, 460, 521, 519, 492, 492, 504, 488, 480, 499, 529, 496, 523, 513, 530, 483, 537, 553, 516, 529, 549, 511, 506, 498, 550, 529, 478, 534, 498, 514, 512, 502, 469, 538, 517, 510, 525] payload_rows_per_dst=[10938, 11072, 10938, 11194, 10660, 10862, 11150, 10652] dup_rows_per_dst=[5392, 5564, 5380, 5772, 5050, 5426, 5696, 5326] combine_rows_per_src=[10938, 11072, 10938, 11194, 10660, 10862, 11150, 10652] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625770496 dispatch_meta_bytes=458536 combine_payload_bytes=625770496 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626229032 ingress_bytes=776765440 link_bytes=776765440 token_bytes=1251999528 weight_movement_bytes=150994944 total_nvlink_bytes=1402994472 weight_to_token_ratio=0.120603 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850479159e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627638272 dispatch_meta_bytes=458544 combine_payload_bytes=627638272 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628096816 ingress_bytes=703135744 link_bytes=703135744 token_bytes=1255735088 weight_movement_bytes=75497472 total_nvlink_bytes=1331232560 weight_to_token_ratio=0.060122 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916750972e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628785152 dispatch_meta_bytes=458480 combine_payload_bytes=628785152 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=629243632 ingress_bytes=930775040 link_bytes=930775040 token_bytes=1258028784 weight_movement_bytes=301989888 total_nvlink_bytes=1560018672 weight_to_token_ratio=0.240050 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.180353767e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87400 payload_rows_remote=76410 dup_rows=43672 dup_rows_remote=37936 dedup_ratio=0.333191 router_tokens_per_expert=[515, 518, 514, 513, 519, 504, 500, 528, 486, 508, 502, 525, 502, 501, 527, 505, 493, 511, 532, 487, 526, 490, 489, 508, 493, 534, 551, 551, 494, 457, 495, 485, 498, 546, 491, 518, 521, 526, 528, 533, 541, 520, 484, 480, 553, 493, 521, 527, 507, 528, 507, 475, 503, 497, 509, 520, 535, 513, 494, 505, 516, 475, 462, 508, 537, 502, 515, 513, 516, 472, 519, 541, 506, 514, 547, 532, 524, 525, 549, 551, 513, 503, 499, 505, 532, 525, 544, 486, 466, 480, 521, 539, 517, 538, 534, 477, 462, 503, 566, 497, 530, 500, 537, 496, 508, 531, 536, 491, 533, 562, 513, 510, 511, 527, 543, 495, 505, 500, 461, 526, 504, 504, 490, 530, 483, 521, 511, 511] payload_rows_per_dst=[10910, 10886, 11064, 10778, 10990, 10860, 11034, 10878] dup_rows_per_dst=[5424, 5306, 5496, 5330, 5736, 5498, 5516, 5366] combine_rows_per_src=[10910, 10886, 11064, 10778, 10990, 10860, 11034, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=458968 combine_payload_bytes=626917376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627376344 ingress_bytes=702414848 link_bytes=702414848 token_bytes=1254293720 weight_movement_bytes=75497472 total_nvlink_bytes=1329791192 weight_to_token_ratio=0.060191 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907608996e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625950720 dispatch_meta_bytes=457384 combine_payload_bytes=625950720 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626408104 ingress_bytes=776945664 link_bytes=776945664 token_bytes=1252358824 weight_movement_bytes=150994944 total_nvlink_bytes=1403353768 weight_to_token_ratio=0.120568 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.852764653e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87726 payload_rows_remote=76672 dup_rows=43346 dup_rows_remote=37752 dedup_ratio=0.330704 router_tokens_per_expert=[530, 537, 527, 499, 493, 536, 486, 477, 529, 537, 513, 515, 503, 531, 520, 513, 468, 544, 513, 559, 480, 490, 532, 510, 512, 491, 519, 514, 525, 555, 494, 527, 511, 506, 511, 529, 549, 528, 465, 472, 557, 530, 506, 502, 488, 529, 537, 511, 496, 471, 492, 515, 465, 520, 499, 535, 537, 515, 518, 569, 503, 480, 534, 490, 486, 483, 486, 544, 515, 515, 512, 488, 559, 486, 485, 502, 508, 482, 489, 501, 500, 530, 522, 512, 494, 529, 503, 511, 499, 541, 484, 533, 504, 485, 516, 518, 539, 505, 514, 524, 554, 503, 531, 509, 508, 527, 528, 519, 516, 498, 548, 501, 495, 497, 508, 538, 528, 518, 471, 503, 531, 498, 512, 499, 515, 502, 514, 512] payload_rows_per_dst=[11002, 10992, 11030, 10908, 10766, 10990, 11054, 10984] dup_rows_per_dst=[5490, 5474, 5432, 5370, 5316, 5372, 5594, 5298] combine_rows_per_src=[11002, 10992, 11030, 10908, 10766, 10990, 11054, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87556 payload_rows_remote=76578 dup_rows=43516 dup_rows_remote=38014 dedup_ratio=0.332001 router_tokens_per_expert=[528, 504, 486, 528, 539, 506, 450, 514, 432, 519, 518, 503, 535, 529, 508, 534, 556, 568, 509, 520, 488, 486, 533, 495, 500, 525, 499, 511, 536, 508, 495, 511, 478, 502, 545, 529, 500, 522, 509, 497, 499, 489, 509, 498, 541, 523, 509, 516, 500, 510, 520, 490, 485, 508, 509, 469, 535, 497, 505, 491, 480, 551, 477, 551, 515, 503, 522, 519, 535, 499, 491, 473, 511, 523, 513, 520, 539, 475, 515, 516, 503, 510, 504, 524, 519, 514, 485, 526, 491, 532, 518, 509, 563, 514, 495, 522, 467, 544, 524, 490, 499, 533, 532, 522, 502, 510, 529, 522, 507, 508, 541, 517, 526, 530, 498, 495, 530, 521, 540, 482, 531, 474, 536, 521, 529, 559, 513, 489] payload_rows_per_dst=[10820, 10978, 10908, 10834, 10924, 11036, 11014, 11042] dup_rows_per_dst=[5446, 5502, 5424, 5322, 5414, 5422, 5480, 5506] combine_rows_per_src=[10820, 10978, 10908, 10834, 10924, 11036, 11014, 11042] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87592 payload_rows_remote=76622 dup_rows=43480 dup_rows_remote=38002 dedup_ratio=0.331726 router_tokens_per_expert=[526, 526, 536, 528, 512, 521, 514, 494, 514, 505, 476, 512, 495, 483, 478, 537, 505, 520, 503, 523, 508, 513, 526, 514, 537, 524, 487, 515, 506, 540, 527, 505, 521, 516, 543, 506, 516, 518, 516, 545, 522, 507, 519, 531, 483, 519, 473, 489, 516, 507, 489, 496, 489, 510, 543, 463, 491, 499, 509, 480, 528, 508, 508, 499, 484, 521, 542, 497, 527, 527, 512, 520, 512, 488, 531, 544, 526, 500, 500, 469, 498, 508, 509, 492, 516, 467, 499, 523, 478, 516, 571, 501, 503, 553, 491, 516, 505, 556, 526, 486, 518, 520, 509, 502, 493, 556, 502, 534, 493, 486, 499, 508, 513, 538, 516, 538, 525, 550, 471, 543, 502, 510, 520, 505, 561, 491, 495, 555] payload_rows_per_dst=[11016, 10930, 10970, 10748, 10956, 10908, 11004, 11060] dup_rows_per_dst=[5298, 5576, 5478, 5322, 5444, 5374, 5382, 5606] combine_rows_per_src=[11016, 10930, 10970, 10748, 10956, 10908, 11004, 11060] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87348 payload_rows_remote=76446 dup_rows=43724 dup_rows_remote=38372 dedup_ratio=0.333588 router_tokens_per_expert=[500, 523, 475, 497, 533, 544, 468, 523, 524, 504, 553, 492, 532, 516, 508, 569, 493, 534, 506, 523, 538, 504, 516, 506, 523, 515, 506, 521, 505, 526, 496, 520, 522, 540, 500, 503, 519, 525, 507, 543, 507, 502, 495, 494, 519, 489, 529, 527, 527, 483, 513, 531, 528, 527, 511, 512, 484, 477, 503, 486, 506, 512, 508, 506, 534, 490, 523, 503, 495, 526, 515, 514, 500, 499, 487, 523, 490, 513, 515, 533, 499, 535, 524, 523, 522, 475, 500, 515, 515, 520, 519, 525, 508, 521, 539, 535, 503, 552, 505, 539, 533, 521, 519, 510, 485, 522, 514, 487, 544, 491, 504, 487, 461, 512, 545, 510, 492, 470, 545, 515, 475, 516, 489, 488, 546, 467, 512, 514] payload_rows_per_dst=[10806, 10882, 10830, 10912, 10850, 11150, 11016, 10902] dup_rows_per_dst=[5268, 5538, 5404, 5316, 5470, 5848, 5528, 5352] combine_rows_per_src=[10806, 10882, 10830, 10912, 10850, 11150, 11016, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76646 dup_rows=43546 dup_rows_remote=38080 dedup_ratio=0.332230 router_tokens_per_expert=[510, 529, 486, 530, 497, 515, 510, 521, 521, 518, 543, 509, 510, 484, 521, 524, 508, 540, 490, 549, 472, 497, 537, 502, 520, 537, 508, 530, 563, 481, 526, 491, 527, 525, 548, 521, 532, 458, 483, 556, 486, 534, 524, 527, 472, 517, 494, 531, 519, 507, 451, 484, 539, 499, 542, 512, 530, 512, 496, 470, 548, 506, 501, 557, 499, 490, 503, 485, 517, 516, 529, 548, 536, 523, 491, 479, 478, 497, 544, 526, 460, 517, 501, 567, 498, 522, 517, 519, 524, 491, 534, 544, 544, 484, 471, 507, 507, 504, 514, 497, 534, 491, 495, 484, 485, 509, 495, 501, 530, 529, 530, 514, 527, 508, 501, 523, 515, 505, 516, 481, 491, 502, 481, 509, 567, 499, 526, 518] payload_rows_per_dst=[10984, 10986, 10988, 10880, 10968, 10894, 10884, 10942] dup_rows_per_dst=[5472, 5516, 5482, 5466, 5354, 5506, 5354, 5396] combine_rows_per_src=[10984, 10986, 10988, 10880, 10968, 10894, 10884, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628097024 dispatch_meta_bytes=457696 combine_payload_bytes=628097024 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628554720 ingress_bytes=703594496 link_bytes=703594496 token_bytes=1256651744 weight_movement_bytes=75497472 total_nvlink_bytes=1332149216 weight_to_token_ratio=0.060078 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.922568593e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87434 payload_rows_remote=76520 dup_rows=43638 dup_rows_remote=38088 dedup_ratio=0.332932 router_tokens_per_expert=[518, 482, 505, 496, 493, 494, 509, 508, 526, 506, 515, 544, 527, 508, 512, 499, 473, 519, 470, 519, 519, 520, 523, 510, 478, 532, 496, 541, 511, 503, 480, 487, 515, 491, 514, 508, 537, 537, 479, 515, 543, 508, 531, 533, 530, 534, 497, 519, 519, 517, 523, 509, 515, 530, 543, 485, 487, 556, 519, 529, 493, 499, 510, 559, 452, 539, 518, 548, 493, 475, 518, 506, 515, 504, 536, 522, 495, 498, 502, 494, 526, 491, 513, 510, 539, 514, 491, 525, 493, 542, 550, 492, 535, 513, 503, 495, 531, 503, 468, 448, 517, 516, 482, 477, 519, 519, 478, 535, 477, 532, 520, 484, 509, 524, 525, 544, 508, 477, 536, 554, 502, 556, 525, 552, 477, 548, 500, 539] payload_rows_per_dst=[10906, 10770, 11040, 11004, 10846, 10914, 10840, 11114] dup_rows_per_dst=[5378, 5392, 5542, 5582, 5384, 5550, 5172, 5638] combine_rows_per_src=[10906, 10770, 11040, 11004, 10846, 10914, 10840, 11114] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=458368 combine_payload_bytes=627326976 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=627785344 ingress_bytes=702824448 link_bytes=702824448 token_bytes=1255112320 weight_movement_bytes=75497472 total_nvlink_bytes=1330609792 weight_to_token_ratio=0.060152 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.912803301e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627687424 dispatch_meta_bytes=458496 combine_payload_bytes=627687424 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=628145920 ingress_bytes=929677312 link_bytes=929677312 token_bytes=1255833344 weight_movement_bytes=301989888 total_nvlink_bytes=1557823232 weight_to_token_ratio=0.240470 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178961694e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87184 payload_rows_remote=76166 dup_rows=43888 dup_rows_remote=38162 dedup_ratio=0.334839 router_tokens_per_expert=[497, 495, 539, 446, 530, 506, 534, 526, 546, 530, 543, 519, 532, 546, 523, 502, 536, 515, 515, 542, 496, 553, 540, 513, 472, 505, 497, 515, 533, 479, 488, 485, 534, 562, 509, 518, 495, 498, 514, 499, 486, 519, 516, 522, 512, 499, 502, 499, 550, 482, 505, 489, 545, 500, 492, 532, 532, 502, 523, 503, 545, 525, 504, 483, 473, 496, 507, 520, 463, 542, 492, 501, 503, 518, 527, 532, 504, 503, 520, 520, 539, 491, 504, 501, 502, 505, 517, 551, 494, 510, 513, 561, 510, 543, 492, 508, 509, 452, 494, 522, 519, 476, 487, 493, 495, 498, 510, 555, 520, 514, 529, 529, 501, 527, 523, 498, 501, 558, 511, 543, 521, 538, 515, 484, 495, 486, 484, 493] payload_rows_per_dst=[11018, 10826, 10856, 11052, 11176, 10690, 10902, 10664] dup_rows_per_dst=[5726, 5542, 5396, 5866, 5876, 4982, 5302, 5198] combine_rows_per_src=[11018, 10826, 10856, 11052, 11176, 10690, 10902, 10664] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626245632 dispatch_meta_bytes=459272 combine_payload_bytes=626245632 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626704904 ingress_bytes=777240576 link_bytes=777240576 token_bytes=1252950536 weight_movement_bytes=150994944 total_nvlink_bytes=1403945480 weight_to_token_ratio=0.120511 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.856504553e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627884032 dispatch_meta_bytes=458904 combine_payload_bytes=627884032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628342936 ingress_bytes=703381504 link_bytes=703381504 token_bytes=1256226968 weight_movement_bytes=75497472 total_nvlink_bytes=1331724440 weight_to_token_ratio=0.060099 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919867555e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87458 payload_rows_remote=76550 dup_rows=43614 dup_rows_remote=38194 dedup_ratio=0.332748 router_tokens_per_expert=[483, 539, 523, 500, 547, 526, 498, 540, 499, 498, 512, 521, 518, 520, 506, 491, 479, 482, 479, 500, 502, 542, 523, 526, 509, 514, 485, 514, 503, 535, 503, 488, 505, 486, 509, 519, 514, 536, 514, 479, 485, 491, 528, 496, 528, 475, 548, 533, 524, 481, 520, 518, 521, 559, 540, 515, 494, 497, 485, 505, 546, 482, 542, 516, 527, 503, 525, 488, 508, 515, 488, 501, 517, 527, 532, 523, 499, 507, 538, 466, 513, 493, 489, 475, 551, 498, 485, 516, 557, 535, 539, 536, 499, 503, 503, 526, 515, 506, 509, 533, 518, 558, 521, 486, 521, 523, 481, 532, 547, 516, 496, 511, 512, 509, 544, 517, 510, 536, 513, 511, 517, 500, 466, 544, 469, 531, 515, 491] payload_rows_per_dst=[11034, 10806, 10874, 11002, 10908, 10950, 11018, 10866] dup_rows_per_dst=[5408, 5362, 5418, 5488, 5420, 5486, 5528, 5504] combine_rows_per_src=[11034, 10806, 10874, 11002, 10908, 10950, 11018, 10866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=458432 combine_payload_bytes=626851840 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627310272 ingress_bytes=777846784 link_bytes=777846784 token_bytes=1254162112 weight_movement_bytes=150994944 total_nvlink_bytes=1405157056 weight_to_token_ratio=0.120395 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864192123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=623951872 dispatch_meta_bytes=457312 combine_payload_bytes=623951872 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624409184 ingress_bytes=774946816 link_bytes=774946816 token_bytes=1248361056 weight_movement_bytes=150994944 total_nvlink_bytes=1399356000 weight_to_token_ratio=0.120955 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.827416447e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627097600 dispatch_meta_bytes=458976 combine_payload_bytes=627097600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627556576 ingress_bytes=702595072 link_bytes=702595072 token_bytes=1254654176 weight_movement_bytes=75497472 total_nvlink_bytes=1330151648 weight_to_token_ratio=0.060174 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909894490e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87114 payload_rows_remote=76214 dup_rows=43958 dup_rows_remote=38446 dedup_ratio=0.335373 router_tokens_per_expert=[525, 477, 508, 522, 548, 505, 493, 500, 517, 503, 528, 539, 513, 512, 510, 556, 501, 503, 532, 504, 576, 543, 503, 519, 514, 500, 508, 459, 548, 511, 490, 514, 552, 518, 527, 487, 488, 524, 520, 491, 497, 535, 534, 468, 528, 504, 503, 507, 492, 530, 533, 488, 541, 518, 499, 529, 490, 507, 533, 489, 515, 516, 522, 557, 560, 549, 505, 511, 514, 496, 486, 498, 490, 478, 541, 523, 496, 475, 549, 498, 517, 463, 524, 520, 494, 555, 501, 495, 512, 478, 526, 477, 519, 476, 496, 496, 521, 502, 531, 503, 507, 517, 511, 523, 528, 515, 487, 500, 533, 486, 513, 529, 539, 496, 539, 560, 483, 504, 491, 520, 545, 482, 528, 502, 477, 522, 513, 488] payload_rows_per_dst=[10988, 10862, 10882, 10916, 10806, 10808, 10900, 10952] dup_rows_per_dst=[5524, 5588, 5484, 5602, 5532, 5290, 5512, 5426] combine_rows_per_src=[10988, 10862, 10882, 10916, 10806, 10808, 10900, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87410 payload_rows_remote=76416 dup_rows=43662 dup_rows_remote=38152 dedup_ratio=0.333115 router_tokens_per_expert=[518, 483, 528, 508, 531, 523, 502, 535, 532, 533, 539, 545, 484, 517, 510, 520, 505, 522, 524, 528, 503, 508, 511, 538, 485, 538, 515, 540, 503, 489, 521, 522, 547, 512, 516, 497, 533, 498, 526, 506, 493, 471, 508, 516, 448, 506, 497, 494, 503, 555, 532, 508, 499, 502, 517, 512, 527, 484, 511, 548, 508, 512, 510, 512, 523, 531, 514, 505, 517, 521, 477, 541, 500, 521, 511, 530, 518, 524, 493, 496, 515, 511, 473, 478, 537, 534, 517, 517, 505, 498, 535, 524, 510, 503, 510, 526, 533, 457, 472, 499, 556, 516, 486, 531, 489, 545, 493, 535, 513, 515, 498, 489, 523, 512, 505, 464, 520, 488, 512, 511, 478, 536, 498, 525, 525, 491, 521, 517] payload_rows_per_dst=[11036, 10994, 10842, 10930, 11030, 10922, 10826, 10830] dup_rows_per_dst=[5580, 5510, 5294, 5550, 5414, 5464, 5428, 5422] combine_rows_per_src=[11036, 10994, 10842, 10930, 11030, 10922, 10826, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87048 payload_rows_remote=76122 dup_rows=44024 dup_rows_remote=38582 dedup_ratio=0.335876 router_tokens_per_expert=[523, 514, 555, 552, 493, 539, 521, 513, 496, 495, 506, 579, 541, 515, 502, 526, 525, 519, 535, 466, 523, 540, 535, 540, 525, 491, 497, 486, 542, 484, 484, 499, 468, 506, 528, 504, 512, 507, 539, 487, 512, 514, 512, 533, 543, 488, 515, 516, 533, 519, 489, 505, 523, 495, 507, 506, 540, 522, 550, 510, 508, 513, 531, 505, 520, 489, 513, 494, 542, 488, 499, 480, 511, 512, 566, 493, 494, 517, 500, 512, 533, 501, 515, 475, 562, 521, 539, 476, 496, 525, 516, 512, 484, 550, 516, 456, 528, 497, 519, 476, 550, 525, 521, 530, 534, 497, 495, 491, 513, 519, 507, 489, 510, 509, 488, 526, 487, 510, 504, 491, 507, 483, 466, 459, 520, 484, 551, 542] payload_rows_per_dst=[10990, 10866, 10926, 10886, 10826, 10926, 10860, 10768] dup_rows_per_dst=[5750, 5516, 5442, 5626, 5434, 5428, 5522, 5306] combine_rows_per_src=[10990, 10866, 10926, 10886, 10826, 10926, 10860, 10768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87428 payload_rows_remote=76376 dup_rows=43644 dup_rows_remote=37862 dedup_ratio=0.332977 router_tokens_per_expert=[536, 526, 529, 506, 473, 498, 502, 492, 506, 516, 498, 527, 557, 534, 505, 528, 508, 478, 498, 500, 490, 512, 523, 522, 513, 539, 506, 512, 509, 485, 506, 512, 484, 571, 528, 523, 519, 521, 480, 506, 527, 495, 525, 535, 518, 515, 503, 542, 513, 525, 482, 518, 527, 474, 527, 507, 496, 533, 480, 489, 503, 528, 491, 507, 513, 519, 484, 528, 517, 516, 510, 481, 505, 558, 507, 515, 494, 545, 515, 515, 554, 496, 532, 526, 511, 513, 488, 506, 515, 519, 533, 517, 524, 499, 498, 502, 552, 520, 544, 508, 480, 488, 512, 530, 512, 552, 502, 509, 503, 524, 493, 461, 486, 530, 524, 500, 512, 506, 496, 517, 471, 526, 557, 490, 485, 515, 534, 504] payload_rows_per_dst=[10742, 10922, 10980, 10884, 10950, 10924, 10974, 11052] dup_rows_per_dst=[5126, 5304, 5580, 5220, 5518, 5608, 5506, 5782] combine_rows_per_src=[10742, 10922, 10980, 10884, 10950, 10924, 10974, 11052] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87458 payload_rows_remote=76594 dup_rows=43614 dup_rows_remote=38106 dedup_ratio=0.332748 router_tokens_per_expert=[481, 518, 507, 513, 487, 525, 512, 515, 501, 518, 497, 489, 517, 517, 497, 516, 521, 508, 499, 522, 491, 533, 477, 486, 520, 511, 520, 532, 500, 526, 534, 480, 532, 509, 505, 493, 533, 551, 531, 518, 508, 518, 513, 528, 521, 508, 493, 494, 524, 485, 477, 507, 500, 472, 496, 519, 553, 519, 511, 527, 527, 517, 513, 539, 535, 523, 514, 532, 518, 511, 488, 482, 506, 489, 526, 512, 559, 525, 513, 508, 501, 485, 506, 520, 493, 506, 517, 525, 521, 518, 515, 519, 516, 503, 510, 520, 523, 522, 531, 508, 524, 526, 480, 533, 519, 537, 505, 485, 560, 514, 515, 481, 500, 523, 518, 539, 523, 488, 474, 500, 518, 498, 509, 528, 513, 558, 454, 503] payload_rows_per_dst=[10896, 10896, 10952, 10864, 11028, 10886, 11008, 10928] dup_rows_per_dst=[5324, 5424, 5558, 5508, 5454, 5464, 5518, 5364] combine_rows_per_src=[10896, 10896, 10952, 10864, 11028, 10886, 11008, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87442 payload_rows_remote=76434 dup_rows=43630 dup_rows_remote=38254 dedup_ratio=0.332870 router_tokens_per_expert=[534, 523, 511, 508, 516, 475, 539, 514, 527, 464, 496, 497, 539, 485, 502, 519, 509, 529, 475, 544, 509, 547, 528, 540, 498, 507, 528, 550, 495, 531, 529, 465, 478, 528, 461, 509, 488, 507, 522, 514, 498, 469, 531, 539, 518, 525, 519, 502, 485, 557, 492, 545, 524, 513, 527, 542, 501, 509, 492, 513, 487, 488, 517, 570, 497, 498, 513, 494, 515, 515, 531, 538, 517, 500, 527, 540, 536, 503, 542, 506, 522, 504, 534, 510, 506, 501, 540, 509, 527, 508, 532, 484, 474, 505, 521, 515, 486, 522, 538, 504, 499, 488, 506, 499, 491, 521, 495, 541, 504, 506, 506, 485, 517, 497, 528, 518, 529, 512, 516, 505, 496, 486, 532, 536, 496, 475, 526, 509] payload_rows_per_dst=[10914, 11024, 10914, 10940, 11024, 11008, 10798, 10820] dup_rows_per_dst=[5384, 5544, 5302, 5584, 5520, 5376, 5384, 5536] combine_rows_per_src=[10914, 11024, 10914, 10940, 11024, 11008, 10798, 10820] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=624345088 dispatch_meta_bytes=458640 combine_payload_bytes=624345088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624803728 ingress_bytes=699842560 link_bytes=699842560 token_bytes=1249148816 weight_movement_bytes=75497472 total_nvlink_bytes=1324646288 weight_to_token_ratio=0.060439 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.874988764e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87244 payload_rows_remote=76322 dup_rows=43828 dup_rows_remote=38462 dedup_ratio=0.334381 router_tokens_per_expert=[518, 504, 501, 471, 512, 512, 516, 507, 509, 547, 465, 486, 547, 511, 519, 518, 499, 493, 498, 516, 506, 538, 484, 521, 523, 537, 517, 529, 464, 497, 511, 525, 563, 519, 513, 515, 510, 536, 536, 518, 515, 516, 539, 502, 484, 508, 478, 513, 515, 533, 471, 466, 531, 518, 539, 492, 489, 468, 539, 475, 531, 531, 503, 520, 485, 503, 541, 522, 524, 511, 513, 507, 507, 487, 512, 509, 523, 492, 507, 499, 518, 535, 514, 549, 533, 502, 570, 466, 502, 516, 553, 515, 529, 544, 519, 477, 506, 535, 518, 547, 497, 506, 502, 506, 493, 510, 490, 520, 521, 490, 512, 519, 549, 542, 528, 494, 536, 495, 504, 502, 466, 474, 559, 508, 495, 522, 498, 521] payload_rows_per_dst=[10922, 10820, 11034, 10914, 10812, 10962, 10892, 10888] dup_rows_per_dst=[5366, 5550, 5494, 5328, 5418, 5722, 5452, 5498] combine_rows_per_src=[10922, 10820, 11034, 10914, 10812, 10962, 10892, 10888] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625672192 dispatch_meta_bytes=456952 combine_payload_bytes=625672192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626129144 ingress_bytes=701169664 link_bytes=701169664 token_bytes=1251801336 weight_movement_bytes=75497472 total_nvlink_bytes=1327298808 weight_to_token_ratio=0.060311 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.891818310e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625999872 dispatch_meta_bytes=458272 combine_payload_bytes=625999872 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626458144 ingress_bytes=701497344 link_bytes=701497344 token_bytes=1252458016 weight_movement_bytes=75497472 total_nvlink_bytes=1327955488 weight_to_token_ratio=0.060279 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895973754e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=623591424 dispatch_meta_bytes=458816 combine_payload_bytes=623591424 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=624050240 ingress_bytes=774586368 link_bytes=774586368 token_bytes=1247641664 weight_movement_bytes=150994944 total_nvlink_bytes=1398636608 weight_to_token_ratio=0.121024 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.822845459e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87498 payload_rows_remote=76576 dup_rows=43574 dup_rows_remote=38190 dedup_ratio=0.332443 router_tokens_per_expert=[498, 533, 524, 540, 544, 495, 515, 516, 527, 523, 490, 492, 506, 530, 535, 497, 510, 498, 477, 532, 548, 488, 523, 463, 500, 499, 511, 535, 498, 520, 526, 498, 514, 480, 548, 530, 481, 502, 570, 519, 499, 515, 558, 506, 483, 513, 499, 477, 504, 484, 529, 526, 520, 530, 483, 535, 483, 504, 530, 515, 520, 506, 504, 487, 533, 491, 526, 485, 530, 537, 531, 489, 496, 478, 503, 532, 492, 497, 552, 481, 473, 474, 488, 529, 505, 475, 514, 493, 529, 509, 509, 534, 524, 513, 532, 541, 565, 479, 527, 502, 536, 487, 499, 521, 513, 523, 496, 541, 497, 549, 517, 494, 533, 532, 515, 490, 534, 512, 544, 490, 512, 501, 507, 505, 517, 535, 481, 542] payload_rows_per_dst=[11122, 10892, 10886, 10846, 10922, 10858, 11014, 10958] dup_rows_per_dst=[5408, 5360, 5502, 5474, 5384, 5426, 5478, 5542] combine_rows_per_src=[11122, 10892, 10886, 10846, 10922, 10858, 11014, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627458048 dispatch_meta_bytes=458800 combine_payload_bytes=627458048 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627916848 ingress_bytes=778452992 link_bytes=778452992 token_bytes=1255374896 weight_movement_bytes=150994944 total_nvlink_bytes=1406369840 weight_to_token_ratio=0.120279 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.871879694e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626147328 dispatch_meta_bytes=458752 combine_payload_bytes=626147328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626606080 ingress_bytes=701644800 link_bytes=701644800 token_bytes=1252753408 weight_movement_bytes=75497472 total_nvlink_bytes=1328250880 weight_to_token_ratio=0.060265 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897843704e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625229824 dispatch_meta_bytes=459136 combine_payload_bytes=625229824 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=625688960 ingress_bytes=927219712 link_bytes=927219712 token_bytes=1250918784 weight_movement_bytes=301989888 total_nvlink_bytes=1552908672 weight_to_token_ratio=0.241414 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.175845111e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627310592 dispatch_meta_bytes=459064 combine_payload_bytes=627310592 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627769656 ingress_bytes=778305536 link_bytes=778305536 token_bytes=1255080248 weight_movement_bytes=150994944 total_nvlink_bytes=1406075192 weight_to_token_ratio=0.120307 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.870009744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87254 payload_rows_remote=76438 dup_rows=43818 dup_rows_remote=38402 dedup_ratio=0.334305 router_tokens_per_expert=[512, 498, 490, 518, 473, 513, 563, 503, 531, 466, 539, 523, 538, 554, 467, 503, 499, 514, 476, 511, 529, 515, 515, 482, 496, 519, 554, 486, 536, 509, 518, 518, 524, 545, 491, 559, 540, 553, 512, 502, 506, 500, 551, 486, 491, 500, 527, 477, 471, 510, 510, 567, 531, 529, 496, 539, 507, 537, 496, 540, 531, 501, 491, 499, 498, 526, 492, 512, 496, 537, 496, 544, 514, 500, 538, 508, 523, 509, 504, 520, 545, 509, 487, 510, 522, 527, 539, 497, 500, 514, 461, 510, 475, 472, 532, 540, 513, 520, 532, 491, 494, 532, 488, 498, 563, 471, 525, 491, 454, 513, 506, 525, 529, 504, 505, 534, 464, 531, 509, 503, 533, 511, 504, 489, 535, 497, 510, 518] payload_rows_per_dst=[10900, 10854, 11048, 10914, 10914, 10912, 10816, 10896] dup_rows_per_dst=[5482, 5500, 5480, 5596, 5520, 5368, 5416, 5456] combine_rows_per_src=[10900, 10854, 11048, 10914, 10914, 10912, 10816, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87608 payload_rows_remote=76802 dup_rows=43464 dup_rows_remote=38110 dedup_ratio=0.331604 router_tokens_per_expert=[487, 514, 501, 499, 516, 510, 519, 498, 519, 496, 516, 564, 519, 509, 528, 518, 523, 506, 483, 502, 507, 524, 501, 537, 497, 515, 544, 530, 541, 495, 520, 537, 518, 525, 521, 500, 506, 530, 474, 513, 498, 520, 548, 539, 506, 494, 499, 510, 552, 506, 533, 508, 481, 492, 507, 499, 523, 525, 495, 481, 506, 519, 513, 479, 533, 517, 503, 534, 462, 529, 517, 494, 518, 535, 499, 529, 499, 483, 520, 531, 516, 516, 532, 504, 554, 575, 462, 526, 511, 510, 484, 506, 481, 501, 518, 516, 538, 502, 499, 504, 547, 531, 486, 499, 531, 553, 536, 520, 495, 515, 512, 488, 498, 543, 538, 499, 507, 518, 504, 498, 500, 486, 505, 519, 475, 483, 498, 499] payload_rows_per_dst=[11036, 11078, 10980, 10920, 10896, 10944, 10948, 10806] dup_rows_per_dst=[5390, 5446, 5388, 5318, 5424, 5568, 5576, 5354] combine_rows_per_src=[11036, 11078, 10980, 10920, 10896, 10944, 10948, 10806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87218 payload_rows_remote=76352 dup_rows=43854 dup_rows_remote=38318 dedup_ratio=0.334579 router_tokens_per_expert=[521, 474, 519, 483, 472, 519, 576, 499, 548, 515, 524, 522, 487, 519, 506, 497, 491, 504, 491, 522, 531, 531, 513, 502, 533, 526, 499, 509, 502, 528, 512, 507, 509, 516, 504, 506, 497, 536, 499, 506, 542, 484, 512, 513, 542, 488, 535, 522, 529, 511, 495, 533, 485, 522, 469, 472, 536, 526, 501, 525, 507, 519, 531, 548, 522, 536, 538, 527, 514, 525, 514, 504, 499, 521, 468, 515, 528, 519, 521, 567, 502, 515, 521, 520, 527, 529, 509, 501, 502, 513, 515, 517, 469, 537, 522, 462, 512, 537, 511, 539, 498, 497, 509, 543, 503, 485, 513, 514, 480, 510, 496, 520, 509, 545, 517, 503, 495, 500, 494, 500, 473, 541, 503, 483, 500, 468, 539, 518] payload_rows_per_dst=[10950, 10866, 10956, 10902, 10894, 10952, 10836, 10862] dup_rows_per_dst=[5412, 5536, 5466, 5516, 5742, 5370, 5498, 5314] combine_rows_per_src=[10950, 10866, 10956, 10902, 10894, 10952, 10836, 10862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87524 payload_rows_remote=76510 dup_rows=43548 dup_rows_remote=38130 dedup_ratio=0.332245 router_tokens_per_expert=[504, 545, 533, 495, 546, 519, 538, 537, 504, 539, 526, 481, 483, 492, 497, 450, 522, 468, 523, 489, 494, 514, 533, 494, 504, 482, 531, 554, 540, 511, 505, 520, 537, 537, 521, 521, 531, 517, 543, 504, 486, 512, 531, 542, 458, 498, 504, 474, 521, 531, 489, 487, 486, 506, 502, 549, 512, 516, 547, 493, 507, 520, 510, 513, 511, 553, 494, 522, 514, 556, 515, 530, 493, 500, 506, 520, 515, 501, 512, 514, 513, 510, 488, 542, 484, 524, 522, 487, 493, 522, 470, 514, 540, 450, 483, 499, 547, 528, 490, 508, 486, 518, 514, 506, 519, 539, 466, 516, 507, 544, 529, 494, 559, 519, 493, 516, 496, 512, 529, 535, 530, 512, 502, 507, 502, 503, 532, 503] payload_rows_per_dst=[10882, 10976, 11014, 10942, 10944, 10736, 10980, 11050] dup_rows_per_dst=[5496, 5392, 5418, 5436, 5568, 5346, 5442, 5450] combine_rows_per_src=[10882, 10976, 11014, 10942, 10944, 10736, 10980, 11050] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87740 payload_rows_remote=76692 dup_rows=43332 dup_rows_remote=38092 dedup_ratio=0.330597 router_tokens_per_expert=[549, 488, 531, 499, 517, 532, 517, 498, 533, 507, 508, 544, 501, 508, 500, 514, 513, 513, 533, 519, 504, 503, 491, 497, 507, 513, 544, 514, 538, 514, 482, 512, 537, 488, 508, 484, 496, 519, 502, 504, 492, 513, 508, 521, 525, 497, 494, 518, 495, 520, 509, 500, 564, 522, 530, 512, 491, 489, 506, 460, 520, 527, 514, 485, 506, 513, 448, 524, 557, 507, 529, 515, 556, 492, 491, 545, 499, 562, 476, 472, 534, 532, 571, 499, 509, 480, 506, 526, 497, 509, 513, 506, 483, 519, 507, 509, 506, 507, 513, 518, 519, 535, 500, 517, 512, 496, 513, 543, 556, 522, 528, 523, 499, 493, 497, 535, 519, 497, 483, 490, 542, 550, 514, 540, 506, 483, 497, 498] payload_rows_per_dst=[11034, 10940, 10858, 11048, 10898, 11066, 11016, 10880] dup_rows_per_dst=[5458, 5454, 5354, 5240, 5486, 5334, 5600, 5406] combine_rows_per_src=[11034, 10940, 10858, 11048, 10898, 11066, 11016, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76530 dup_rows=43534 dup_rows_remote=37934 dedup_ratio=0.332138 router_tokens_per_expert=[567, 484, 494, 509, 496, 540, 505, 494, 520, 489, 524, 515, 504, 500, 507, 536, 513, 507, 497, 508, 496, 497, 552, 502, 509, 515, 491, 528, 495, 521, 520, 488, 514, 500, 522, 516, 528, 504, 487, 518, 495, 512, 484, 527, 483, 510, 539, 503, 506, 516, 505, 538, 483, 512, 497, 512, 502, 508, 495, 514, 477, 505, 506, 517, 489, 512, 515, 516, 487, 536, 529, 526, 528, 472, 511, 498, 514, 535, 495, 494, 537, 548, 476, 532, 546, 510, 538, 519, 578, 475, 530, 495, 482, 553, 512, 473, 529, 531, 489, 498, 516, 483, 492, 524, 503, 530, 505, 488, 472, 532, 500, 522, 487, 525, 520, 538, 527, 511, 532, 548, 516, 519, 495, 518, 536, 509, 588, 534] payload_rows_per_dst=[10992, 10910, 10916, 10876, 10834, 11008, 10892, 11110] dup_rows_per_dst=[5376, 5368, 5368, 5310, 5480, 5600, 5336, 5696] combine_rows_per_src=[10992, 10910, 10916, 10876, 10834, 11008, 10892, 11110] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626180096 dispatch_meta_bytes=459360 combine_payload_bytes=626180096 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626639456 ingress_bytes=701677568 link_bytes=701677568 token_bytes=1252819552 weight_movement_bytes=75497472 total_nvlink_bytes=1328317024 weight_to_token_ratio=0.060262 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898259248e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87712 payload_rows_remote=76748 dup_rows=43360 dup_rows_remote=38050 dedup_ratio=0.330811 router_tokens_per_expert=[491, 504, 519, 481, 537, 507, 518, 498, 492, 522, 461, 491, 525, 546, 454, 530, 498, 504, 501, 482, 550, 547, 474, 466, 518, 509, 504, 479, 472, 485, 531, 500, 487, 464, 484, 565, 477, 508, 556, 501, 569, 513, 520, 503, 520, 518, 535, 544, 501, 512, 535, 541, 491, 515, 527, 550, 512, 533, 536, 480, 514, 508, 512, 526, 505, 491, 521, 529, 517, 499, 491, 543, 540, 521, 537, 503, 494, 510, 522, 503, 556, 501, 513, 570, 527, 529, 497, 507, 494, 487, 491, 533, 518, 534, 511, 488, 495, 557, 521, 519, 503, 493, 553, 550, 510, 479, 475, 509, 457, 490, 523, 504, 468, 513, 520, 503, 499, 501, 513, 518, 517, 516, 485, 523, 544, 572, 544, 527] payload_rows_per_dst=[10964, 10918, 11006, 11114, 10850, 10972, 10930, 10958] dup_rows_per_dst=[5310, 5456, 5522, 5540, 5146, 5540, 5346, 5500] combine_rows_per_src=[10964, 10918, 11006, 11114, 10850, 10972, 10930, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=629161984 dispatch_meta_bytes=459648 combine_payload_bytes=629161984 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629621632 ingress_bytes=780156928 link_bytes=780156928 token_bytes=1258783616 weight_movement_bytes=150994944 total_nvlink_bytes=1409778560 weight_to_token_ratio=0.119953 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.893488001e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625475584 dispatch_meta_bytes=458680 combine_payload_bytes=625475584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625934264 ingress_bytes=700973056 link_bytes=700973056 token_bytes=1251409848 weight_movement_bytes=75497472 total_nvlink_bytes=1326907320 weight_to_token_ratio=0.060330 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889325044e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626769920 dispatch_meta_bytes=458560 combine_payload_bytes=626769920 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627228480 ingress_bytes=853262336 link_bytes=853262336 token_bytes=1253998400 weight_movement_bytes=226492416 total_nvlink_bytes=1480490816 weight_to_token_ratio=0.180616 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082056748e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87546 payload_rows_remote=76508 dup_rows=43526 dup_rows_remote=38088 dedup_ratio=0.332077 router_tokens_per_expert=[529, 498, 523, 505, 510, 493, 464, 523, 527, 486, 514, 543, 535, 547, 515, 483, 498, 556, 478, 510, 489, 518, 478, 530, 545, 515, 497, 482, 517, 533, 541, 502, 531, 529, 522, 487, 540, 486, 476, 498, 523, 526, 511, 475, 537, 469, 502, 537, 538, 557, 495, 478, 510, 492, 550, 509, 473, 495, 541, 493, 500, 522, 515, 532, 570, 549, 516, 517, 521, 496, 476, 498, 516, 514, 567, 501, 539, 480, 484, 494, 545, 529, 473, 531, 533, 487, 487, 486, 540, 544, 506, 532, 489, 502, 483, 511, 477, 562, 470, 553, 478, 497, 532, 480, 529, 502, 532, 533, 493, 518, 539, 525, 519, 501, 501, 501, 548, 495, 530, 517, 511, 487, 510, 522, 484, 527, 461, 553] payload_rows_per_dst=[10942, 10902, 10906, 10916, 11038, 10884, 11022, 10936] dup_rows_per_dst=[5448, 5476, 5392, 5484, 5438, 5472, 5418, 5398] combine_rows_per_src=[10942, 10902, 10906, 10916, 11038, 10884, 11022, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628260864 dispatch_meta_bytes=459136 combine_payload_bytes=628260864 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628720000 ingress_bytes=703758336 link_bytes=703758336 token_bytes=1256980864 weight_movement_bytes=75497472 total_nvlink_bytes=1332478336 weight_to_token_ratio=0.060063 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924646315e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626933760 dispatch_meta_bytes=457856 combine_payload_bytes=626933760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627391616 ingress_bytes=702431232 link_bytes=702431232 token_bytes=1254325376 weight_movement_bytes=75497472 total_nvlink_bytes=1329822848 weight_to_token_ratio=0.060190 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907816768e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=628719616 dispatch_meta_bytes=459192 combine_payload_bytes=628719616 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629178808 ingress_bytes=704217088 link_bytes=704217088 token_bytes=1257898424 weight_movement_bytes=75497472 total_nvlink_bytes=1333395896 weight_to_token_ratio=0.060019 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.930463936e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=458384 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627211920 ingress_bytes=928743424 link_bytes=928743424 token_bytes=1253965456 weight_movement_bytes=301989888 total_nvlink_bytes=1555955344 weight_to_token_ratio=0.240828 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177777392e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87214 payload_rows_remote=76320 dup_rows=43858 dup_rows_remote=38282 dedup_ratio=0.334610 router_tokens_per_expert=[546, 492, 496, 527, 517, 473, 508, 523, 502, 512, 532, 496, 523, 497, 473, 516, 527, 534, 503, 538, 490, 501, 489, 509, 536, 516, 516, 499, 465, 530, 519, 496, 527, 500, 506, 549, 519, 529, 498, 538, 507, 496, 523, 522, 509, 494, 532, 514, 511, 486, 478, 564, 517, 537, 519, 525, 480, 501, 497, 500, 506, 526, 517, 516, 529, 522, 533, 468, 525, 488, 489, 511, 475, 513, 498, 502, 520, 486, 515, 497, 493, 538, 552, 513, 552, 489, 519, 472, 487, 489, 502, 515, 537, 508, 509, 492, 522, 564, 522, 506, 522, 489, 519, 511, 513, 502, 483, 545, 503, 508, 517, 509, 521, 526, 507, 504, 527, 553, 547, 506, 538, 502, 505, 500, 515, 516, 506, 546] payload_rows_per_dst=[10772, 10892, 10900, 10940, 10864, 10882, 10894, 11070] dup_rows_per_dst=[5494, 5444, 5626, 5420, 5278, 5452, 5576, 5568] combine_rows_per_src=[10772, 10892, 10900, 10940, 10864, 10882, 10894, 11070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87774 payload_rows_remote=76894 dup_rows=43298 dup_rows_remote=37824 dedup_ratio=0.330338 router_tokens_per_expert=[483, 485, 507, 493, 508, 532, 503, 514, 477, 508, 496, 562, 488, 516, 558, 556, 490, 516, 513, 507, 485, 504, 556, 546, 519, 478, 505, 526, 511, 504, 495, 488, 536, 507, 504, 482, 529, 536, 549, 489, 510, 531, 500, 538, 518, 497, 521, 485, 523, 541, 539, 505, 517, 527, 513, 460, 504, 532, 555, 484, 494, 509, 508, 516, 477, 534, 508, 504, 537, 515, 527, 529, 515, 482, 525, 549, 494, 484, 504, 511, 529, 563, 485, 522, 513, 542, 510, 535, 501, 459, 488, 502, 526, 512, 477, 475, 540, 522, 534, 520, 543, 512, 502, 508, 475, 488, 506, 517, 505, 519, 510, 546, 514, 532, 514, 508, 506, 506, 464, 490, 516, 514, 532, 499, 521, 525, 500, 526] payload_rows_per_dst=[11002, 10970, 10936, 11006, 11012, 10980, 10988, 10880] dup_rows_per_dst=[5370, 5316, 5528, 5448, 5378, 5298, 5486, 5474] combine_rows_per_src=[11002, 10970, 10936, 11006, 11012, 10980, 10988, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87594 payload_rows_remote=76728 dup_rows=43478 dup_rows_remote=38080 dedup_ratio=0.331711 router_tokens_per_expert=[516, 496, 512, 504, 492, 499, 526, 502, 541, 502, 554, 517, 527, 481, 511, 534, 469, 519, 500, 483, 498, 524, 506, 528, 551, 564, 487, 502, 492, 482, 517, 510, 491, 492, 527, 517, 519, 525, 482, 471, 518, 488, 536, 479, 509, 500, 543, 501, 555, 541, 509, 498, 497, 471, 540, 494, 471, 527, 508, 547, 522, 542, 487, 487, 507, 475, 523, 528, 508, 503, 488, 507, 485, 525, 502, 501, 516, 561, 519, 561, 478, 489, 529, 507, 485, 529, 513, 503, 545, 560, 511, 524, 546, 482, 513, 542, 529, 499, 518, 480, 470, 535, 506, 510, 526, 519, 513, 494, 514, 505, 503, 497, 519, 496, 482, 540, 549, 532, 506, 522, 533, 505, 522, 482, 509, 541, 528, 547] payload_rows_per_dst=[10926, 10866, 10878, 10914, 11022, 11044, 10972, 10972] dup_rows_per_dst=[5502, 5398, 5318, 5478, 5396, 5468, 5264, 5654] combine_rows_per_src=[10926, 10866, 10878, 10914, 11022, 11044, 10972, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76746 dup_rows=43454 dup_rows_remote=38152 dedup_ratio=0.331528 router_tokens_per_expert=[520, 501, 530, 493, 551, 537, 563, 519, 485, 501, 553, 505, 471, 503, 513, 525, 516, 497, 481, 534, 492, 530, 502, 517, 521, 518, 547, 481, 514, 478, 472, 487, 508, 495, 542, 532, 482, 500, 509, 482, 492, 511, 505, 535, 510, 478, 503, 503, 524, 473, 496, 479, 513, 505, 508, 537, 550, 491, 512, 550, 495, 488, 525, 522, 526, 532, 523, 526, 540, 481, 524, 483, 498, 516, 529, 521, 478, 529, 483, 469, 515, 478, 512, 514, 522, 507, 519, 513, 522, 543, 537, 494, 518, 539, 515, 528, 521, 510, 483, 513, 531, 522, 523, 520, 537, 524, 528, 497, 532, 513, 542, 479, 500, 497, 518, 514, 526, 482, 508, 513, 553, 549, 527, 516, 497, 497, 501, 517] payload_rows_per_dst=[11072, 10886, 10872, 10848, 10938, 11040, 11020, 10942] dup_rows_per_dst=[5468, 5288, 5302, 5488, 5378, 5512, 5530, 5488] combine_rows_per_src=[11072, 10886, 10872, 10848, 10938, 11040, 11020, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87504 payload_rows_remote=76706 dup_rows=43568 dup_rows_remote=38224 dedup_ratio=0.332397 router_tokens_per_expert=[522, 502, 481, 516, 510, 464, 503, 517, 532, 512, 500, 523, 501, 470, 487, 520, 475, 516, 527, 483, 520, 521, 528, 552, 510, 495, 499, 500, 492, 504, 525, 518, 532, 554, 506, 522, 467, 504, 525, 490, 525, 531, 522, 508, 577, 476, 544, 499, 501, 518, 477, 504, 527, 457, 504, 560, 542, 496, 512, 505, 516, 497, 478, 477, 568, 501, 499, 549, 540, 522, 543, 516, 529, 478, 515, 456, 524, 515, 494, 498, 476, 534, 521, 538, 513, 519, 523, 499, 541, 535, 478, 494, 535, 482, 465, 500, 515, 524, 517, 538, 528, 526, 517, 551, 527, 507, 526, 511, 496, 531, 541, 515, 526, 507, 501, 493, 562, 498, 536, 506, 529, 541, 486, 533, 502, 510, 491, 467] payload_rows_per_dst=[10856, 10962, 11020, 10798, 11008, 10900, 11042, 10918] dup_rows_per_dst=[5264, 5368, 5544, 5344, 5486, 5406, 5698, 5458] combine_rows_per_src=[10856, 10962, 11020, 10798, 11008, 10900, 11042, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87470 payload_rows_remote=76642 dup_rows=43602 dup_rows_remote=38142 dedup_ratio=0.332657 router_tokens_per_expert=[496, 511, 509, 503, 489, 516, 529, 537, 518, 509, 540, 510, 524, 542, 507, 501, 519, 488, 517, 504, 497, 509, 539, 557, 511, 521, 496, 527, 530, 512, 516, 515, 516, 512, 516, 536, 513, 509, 489, 506, 504, 533, 537, 525, 540, 510, 529, 543, 479, 518, 519, 497, 479, 535, 498, 521, 541, 534, 511, 512, 537, 528, 519, 505, 483, 519, 512, 529, 459, 523, 507, 529, 541, 519, 522, 528, 513, 526, 467, 508, 516, 519, 504, 517, 488, 513, 540, 490, 504, 524, 492, 512, 504, 515, 496, 510, 518, 506, 488, 495, 520, 515, 534, 541, 495, 477, 480, 479, 542, 490, 518, 486, 515, 523, 470, 503, 487, 566, 491, 501, 487, 517, 491, 469, 545, 510, 529, 469] payload_rows_per_dst=[10978, 11036, 11122, 10988, 10916, 10828, 10830, 10772] dup_rows_per_dst=[5504, 5480, 5514, 5478, 5454, 5460, 5338, 5374] combine_rows_per_src=[10978, 11036, 11122, 10988, 10916, 10828, 10830, 10772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625213440 dispatch_meta_bytes=458408 combine_payload_bytes=625213440 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=625671848 ingress_bytes=851705856 link_bytes=851705856 token_bytes=1250885288 weight_movement_bytes=226492416 total_nvlink_bytes=1477377704 weight_to_token_ratio=0.181066 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080082912e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=629915648 dispatch_meta_bytes=458872 combine_payload_bytes=629915648 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=630374520 ingress_bytes=856408064 link_bytes=856408064 token_bytes=1260290168 weight_movement_bytes=226492416 total_nvlink_bytes=1486782584 weight_to_token_ratio=0.179714 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.086045974e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76418 dup_rows=43534 dup_rows_remote=38022 dedup_ratio=0.332138 router_tokens_per_expert=[495, 533, 468, 534, 528, 486, 526, 517, 487, 514, 514, 522, 521, 512, 495, 479, 533, 526, 493, 502, 502, 505, 538, 490, 524, 504, 513, 442, 522, 465, 527, 509, 515, 511, 515, 464, 494, 529, 501, 526, 513, 514, 529, 497, 491, 504, 508, 469, 537, 486, 490, 527, 505, 515, 511, 519, 501, 483, 512, 498, 493, 493, 528, 538, 526, 520, 498, 536, 514, 523, 521, 533, 484, 522, 505, 514, 466, 501, 512, 514, 483, 552, 507, 525, 505, 530, 494, 495, 508, 496, 543, 480, 487, 489, 531, 523, 519, 504, 555, 525, 510, 488, 512, 508, 536, 503, 521, 543, 496, 529, 497, 561, 507, 530, 561, 521, 508, 501, 539, 569, 499, 528, 553, 503, 514, 565, 515, 537] payload_rows_per_dst=[11120, 11208, 10768, 11076, 10930, 10880, 10784, 10772] dup_rows_per_dst=[5512, 5656, 5190, 5724, 5640, 5236, 5290, 5286] combine_rows_per_src=[11120, 11208, 10768, 11076, 10930, 10880, 10784, 10772] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628555776 dispatch_meta_bytes=459232 combine_payload_bytes=628555776 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629015008 ingress_bytes=704053248 link_bytes=704053248 token_bytes=1257570784 weight_movement_bytes=75497472 total_nvlink_bytes=1333068256 weight_to_token_ratio=0.060034 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.928386215e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87668 payload_rows_remote=76714 dup_rows=43404 dup_rows_remote=37986 dedup_ratio=0.331146 router_tokens_per_expert=[520, 476, 547, 513, 510, 487, 487, 494, 495, 488, 514, 563, 490, 537, 483, 512, 515, 489, 495, 510, 527, 489, 534, 513, 517, 533, 524, 488, 491, 494, 514, 518, 545, 510, 516, 512, 495, 534, 470, 498, 600, 504, 479, 526, 530, 516, 526, 516, 534, 474, 518, 469, 486, 484, 545, 512, 530, 483, 517, 511, 491, 543, 495, 469, 570, 530, 501, 489, 485, 509, 504, 520, 501, 527, 495, 531, 511, 487, 536, 490, 518, 534, 467, 566, 520, 531, 491, 551, 496, 533, 534, 488, 506, 528, 554, 526, 523, 554, 492, 511, 490, 562, 502, 492, 517, 514, 474, 554, 537, 505, 483, 470, 509, 516, 550, 507, 494, 510, 514, 494, 519, 511, 514, 516, 479, 526, 534, 529] payload_rows_per_dst=[10832, 10912, 11058, 10854, 10954, 11104, 10960, 10994] dup_rows_per_dst=[5400, 5390, 5496, 5268, 5418, 5582, 5400, 5450] combine_rows_per_src=[10832, 10912, 11058, 10854, 10954, 11104, 10960, 10994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628703232 dispatch_meta_bytes=459592 combine_payload_bytes=628703232 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629162824 ingress_bytes=779698176 link_bytes=779698176 token_bytes=1257866056 weight_movement_bytes=150994944 total_nvlink_bytes=1408861000 weight_to_token_ratio=0.120041 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.887670380e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628375552 dispatch_meta_bytes=459720 combine_payload_bytes=628375552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628835272 ingress_bytes=703873024 link_bytes=703873024 token_bytes=1257210824 weight_movement_bytes=75497472 total_nvlink_bytes=1332708296 weight_to_token_ratio=0.060052 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926100721e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627851264 dispatch_meta_bytes=459136 combine_payload_bytes=627851264 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628310400 ingress_bytes=778846208 link_bytes=778846208 token_bytes=1256161664 weight_movement_bytes=150994944 total_nvlink_bytes=1407156608 weight_to_token_ratio=0.120203 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876866226e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626016256 dispatch_meta_bytes=457760 combine_payload_bytes=626016256 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626474016 ingress_bytes=701513728 link_bytes=701513728 token_bytes=1252490272 weight_movement_bytes=75497472 total_nvlink_bytes=1327987744 weight_to_token_ratio=0.060278 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.896181526e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628441088 dispatch_meta_bytes=458800 combine_payload_bytes=628441088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628899888 ingress_bytes=703938560 link_bytes=703938560 token_bytes=1257340976 weight_movement_bytes=75497472 total_nvlink_bytes=1332838448 weight_to_token_ratio=0.060045 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926931809e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87688 payload_rows_remote=76634 dup_rows=43384 dup_rows_remote=37848 dedup_ratio=0.330994 router_tokens_per_expert=[537, 495, 524, 499, 530, 500, 523, 485, 505, 487, 532, 524, 527, 515, 494, 468, 499, 539, 535, 537, 482, 538, 519, 519, 513, 537, 537, 533, 504, 531, 508, 494, 500, 489, 533, 494, 497, 543, 515, 489, 514, 519, 513, 519, 494, 520, 483, 512, 511, 543, 515, 537, 509, 497, 531, 526, 516, 461, 483, 508, 506, 476, 495, 537, 527, 522, 517, 483, 501, 492, 510, 492, 514, 508, 509, 501, 493, 516, 518, 500, 542, 516, 485, 509, 517, 501, 534, 518, 494, 541, 519, 543, 507, 483, 521, 500, 508, 497, 546, 516, 507, 496, 518, 536, 550, 534, 555, 543, 488, 495, 469, 537, 487, 505, 493, 529, 489, 490, 487, 513, 515, 525, 549, 520, 506, 531, 504, 510] payload_rows_per_dst=[10932, 11116, 10892, 10888, 10914, 10990, 11054, 10902] dup_rows_per_dst=[5358, 5534, 5376, 5414, 5292, 5470, 5536, 5404] combine_rows_per_src=[10932, 11116, 10892, 10888, 10914, 10990, 11054, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87256 payload_rows_remote=76130 dup_rows=43816 dup_rows_remote=38094 dedup_ratio=0.334290 router_tokens_per_expert=[587, 499, 561, 495, 500, 511, 536, 495, 495, 482, 509, 516, 529, 477, 519, 528, 481, 494, 496, 504, 541, 510, 469, 519, 505, 525, 544, 495, 462, 526, 542, 511, 491, 509, 539, 507, 491, 532, 541, 518, 511, 532, 562, 527, 493, 547, 537, 494, 473, 497, 526, 514, 520, 490, 493, 516, 511, 495, 514, 520, 504, 525, 547, 543, 518, 511, 532, 508, 483, 504, 487, 505, 465, 498, 522, 509, 545, 500, 521, 526, 516, 478, 525, 523, 494, 493, 505, 490, 523, 463, 501, 481, 513, 527, 496, 479, 524, 517, 465, 497, 486, 486, 523, 523, 529, 510, 540, 525, 530, 514, 486, 481, 503, 548, 521, 528, 533, 556, 529, 499, 561, 520, 505, 523, 490, 533, 502, 526] payload_rows_per_dst=[10810, 10506, 10876, 10962, 10998, 10722, 11256, 11126] dup_rows_per_dst=[5206, 4854, 5512, 5594, 5732, 5292, 5904, 5722] combine_rows_per_src=[10810, 10506, 10876, 10962, 10998, 10722, 11256, 11126] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87634 payload_rows_remote=76694 dup_rows=43438 dup_rows_remote=37902 dedup_ratio=0.331406 router_tokens_per_expert=[527, 532, 538, 502, 503, 504, 479, 523, 495, 536, 546, 542, 489, 525, 516, 492, 533, 538, 508, 509, 546, 529, 536, 524, 511, 498, 498, 511, 482, 513, 487, 515, 530, 524, 492, 481, 487, 509, 513, 543, 485, 530, 506, 521, 525, 507, 508, 537, 515, 521, 518, 540, 538, 555, 517, 547, 509, 472, 486, 471, 503, 539, 516, 522, 518, 521, 477, 469, 518, 558, 514, 529, 526, 502, 552, 505, 476, 495, 513, 534, 546, 496, 482, 530, 488, 510, 523, 518, 482, 514, 515, 565, 511, 492, 490, 470, 548, 496, 525, 514, 482, 558, 510, 481, 491, 495, 500, 488, 515, 488, 497, 494, 475, 469, 523, 510, 510, 540, 522, 493, 498, 519, 503, 523, 495, 544, 528, 509] payload_rows_per_dst=[11000, 10940, 10976, 11024, 11000, 10920, 10882, 10892] dup_rows_per_dst=[5498, 5536, 5420, 5514, 5414, 5344, 5282, 5430] combine_rows_per_src=[11000, 10940, 10976, 11024, 11000, 10920, 10882, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87730 payload_rows_remote=76862 dup_rows=43342 dup_rows_remote=37912 dedup_ratio=0.330673 router_tokens_per_expert=[506, 516, 525, 521, 516, 513, 506, 505, 501, 480, 484, 508, 514, 514, 518, 463, 520, 558, 510, 529, 541, 570, 483, 505, 528, 518, 494, 518, 522, 531, 545, 492, 534, 481, 456, 525, 532, 538, 470, 547, 480, 534, 512, 471, 534, 513, 522, 500, 452, 499, 499, 529, 536, 488, 513, 561, 548, 520, 515, 535, 552, 497, 496, 512, 457, 493, 495, 515, 508, 499, 544, 529, 521, 545, 528, 530, 499, 531, 519, 510, 523, 545, 494, 512, 486, 506, 521, 497, 536, 497, 535, 505, 507, 506, 498, 500, 544, 502, 504, 463, 502, 499, 492, 451, 518, 512, 475, 487, 528, 551, 525, 521, 569, 495, 488, 573, 513, 499, 529, 499, 525, 505, 503, 488, 473, 501, 525, 531] payload_rows_per_dst=[10824, 11094, 10868, 11012, 11010, 10988, 11000, 10934] dup_rows_per_dst=[5356, 5634, 5430, 5492, 5436, 5348, 5148, 5498] combine_rows_per_src=[10824, 11094, 10868, 11012, 11010, 10988, 11000, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87584 payload_rows_remote=76650 dup_rows=43488 dup_rows_remote=38088 dedup_ratio=0.331787 router_tokens_per_expert=[541, 460, 542, 524, 552, 532, 522, 519, 524, 483, 512, 509, 536, 529, 503, 532, 462, 460, 532, 527, 525, 521, 524, 547, 492, 562, 506, 501, 467, 525, 535, 535, 508, 517, 477, 489, 532, 526, 512, 540, 521, 527, 532, 511, 473, 545, 492, 565, 517, 503, 527, 471, 502, 516, 500, 504, 529, 499, 514, 481, 493, 557, 526, 528, 555, 506, 509, 512, 507, 504, 524, 517, 491, 516, 493, 506, 505, 487, 473, 527, 519, 500, 506, 519, 516, 474, 494, 515, 482, 556, 505, 518, 542, 525, 531, 504, 526, 563, 514, 490, 514, 520, 522, 470, 518, 503, 463, 500, 487, 531, 478, 490, 541, 521, 537, 563, 510, 478, 475, 495, 512, 492, 478, 520, 481, 499, 509, 523] payload_rows_per_dst=[11084, 11018, 11106, 10934, 10784, 10918, 10836, 10904] dup_rows_per_dst=[5556, 5424, 5428, 5400, 5480, 5494, 5342, 5364] combine_rows_per_src=[11084, 11018, 11106, 10934, 10784, 10918, 10836, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87618 payload_rows_remote=76696 dup_rows=43454 dup_rows_remote=38076 dedup_ratio=0.331528 router_tokens_per_expert=[503, 525, 498, 495, 471, 519, 478, 524, 535, 505, 480, 530, 521, 512, 557, 528, 493, 481, 495, 509, 525, 530, 503, 536, 517, 498, 523, 509, 520, 501, 480, 549, 517, 533, 499, 530, 511, 508, 501, 518, 497, 538, 524, 525, 532, 482, 517, 510, 516, 533, 489, 473, 549, 481, 517, 517, 500, 503, 522, 510, 521, 511, 492, 508, 499, 532, 519, 526, 479, 507, 506, 497, 490, 493, 514, 538, 523, 491, 504, 550, 517, 517, 494, 495, 501, 509, 501, 521, 494, 481, 481, 536, 559, 517, 533, 494, 544, 521, 514, 481, 549, 511, 499, 495, 500, 542, 508, 498, 519, 522, 501, 494, 482, 521, 511, 486, 531, 576, 500, 510, 534, 508, 530, 533, 518, 509, 508, 529] payload_rows_per_dst=[10908, 10894, 11006, 10932, 10960, 10922, 10968, 11028] dup_rows_per_dst=[5454, 5444, 5478, 5352, 5376, 5378, 5428, 5544] combine_rows_per_src=[10908, 10894, 11006, 10932, 10960, 10922, 10968, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627785728 dispatch_meta_bytes=457928 combine_payload_bytes=627785728 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628243656 ingress_bytes=703283200 link_bytes=703283200 token_bytes=1256029384 weight_movement_bytes=75497472 total_nvlink_bytes=1331526856 weight_to_token_ratio=0.060108 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918620922e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=623656960 dispatch_meta_bytes=456896 combine_payload_bytes=623656960 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=624113856 ingress_bytes=850149376 link_bytes=850149376 token_bytes=1247770816 weight_movement_bytes=226492416 total_nvlink_bytes=1474263232 weight_to_token_ratio=0.181518 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.078109076e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87418 payload_rows_remote=76384 dup_rows=43654 dup_rows_remote=37938 dedup_ratio=0.333054 router_tokens_per_expert=[517, 512, 547, 510, 481, 511, 532, 525, 498, 543, 500, 532, 504, 525, 517, 544, 570, 504, 543, 522, 465, 492, 523, 535, 541, 519, 485, 505, 505, 556, 494, 515, 523, 549, 528, 538, 533, 518, 506, 488, 512, 527, 482, 519, 474, 508, 497, 509, 482, 545, 525, 530, 531, 488, 508, 521, 486, 496, 509, 541, 479, 506, 490, 485, 537, 542, 471, 503, 496, 541, 484, 484, 508, 554, 516, 495, 522, 509, 533, 499, 536, 519, 524, 512, 521, 467, 535, 521, 533, 521, 506, 526, 498, 516, 510, 512, 500, 481, 510, 537, 473, 496, 483, 526, 507, 475, 538, 535, 479, 502, 526, 476, 499, 534, 519, 511, 546, 480, 516, 479, 507, 495, 513, 534, 530, 479, 532, 462] payload_rows_per_dst=[11034, 11144, 10964, 10780, 10904, 11064, 10776, 10752] dup_rows_per_dst=[5716, 5568, 5498, 5464, 5484, 5410, 5312, 5202] combine_rows_per_src=[11034, 11144, 10964, 10780, 10904, 11064, 10776, 10752] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87746 payload_rows_remote=76814 dup_rows=43326 dup_rows_remote=37970 dedup_ratio=0.330551 router_tokens_per_expert=[516, 530, 501, 527, 481, 484, 511, 519, 480, 526, 511, 549, 505, 498, 499, 531, 543, 499, 522, 572, 517, 472, 513, 533, 521, 504, 507, 467, 512, 486, 485, 530, 549, 503, 479, 506, 520, 467, 471, 521, 570, 553, 476, 502, 508, 517, 493, 486, 504, 528, 539, 531, 501, 499, 513, 475, 504, 475, 523, 523, 529, 512, 522, 477, 510, 504, 526, 462, 518, 544, 483, 512, 487, 502, 505, 524, 509, 480, 530, 548, 502, 533, 506, 536, 535, 541, 511, 519, 549, 508, 532, 529, 483, 490, 558, 574, 520, 496, 528, 482, 538, 533, 503, 507, 513, 507, 475, 468, 538, 551, 529, 486, 492, 541, 480, 479, 529, 541, 524, 533, 494, 492, 491, 491, 523, 499, 523, 553] payload_rows_per_dst=[10954, 10828, 10970, 11032, 10932, 11142, 10976, 10912] dup_rows_per_dst=[5382, 5538, 5272, 5278, 5356, 5670, 5372, 5458] combine_rows_per_src=[10954, 10828, 10970, 11032, 10932, 11142, 10976, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628277248 dispatch_meta_bytes=458384 combine_payload_bytes=628277248 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628735632 ingress_bytes=779272192 link_bytes=779272192 token_bytes=1257012880 weight_movement_bytes=150994944 total_nvlink_bytes=1408007824 weight_to_token_ratio=0.120122 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.882268303e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629653504 dispatch_meta_bytes=459096 combine_payload_bytes=629653504 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=630112600 ingress_bytes=856145920 link_bytes=856145920 token_bytes=1259766104 weight_movement_bytes=226492416 total_nvlink_bytes=1486258520 weight_to_token_ratio=0.179789 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085713538e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=458952 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628375752 ingress_bytes=703414272 link_bytes=703414272 token_bytes=1256292552 weight_movement_bytes=75497472 total_nvlink_bytes=1331790024 weight_to_token_ratio=0.060095 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920283099e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628293632 dispatch_meta_bytes=459088 combine_payload_bytes=628293632 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=628752720 ingress_bytes=703791104 link_bytes=703791104 token_bytes=1257046352 weight_movement_bytes=75497472 total_nvlink_bytes=1332543824 weight_to_token_ratio=0.060059 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925061860e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625737728 dispatch_meta_bytes=457288 combine_payload_bytes=625737728 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626195016 ingress_bytes=776732672 link_bytes=776732672 token_bytes=1251932744 weight_movement_bytes=150994944 total_nvlink_bytes=1402927688 weight_to_token_ratio=0.120609 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850063615e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629260288 dispatch_meta_bytes=459136 combine_payload_bytes=629260288 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629719424 ingress_bytes=704757760 link_bytes=704757760 token_bytes=1258979712 weight_movement_bytes=75497472 total_nvlink_bytes=1334477184 weight_to_token_ratio=0.059967 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.937320418e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87550 payload_rows_remote=76600 dup_rows=43522 dup_rows_remote=38106 dedup_ratio=0.332047 router_tokens_per_expert=[525, 538, 492, 546, 489, 534, 512, 541, 520, 514, 515, 483, 500, 483, 517, 475, 534, 543, 508, 504, 496, 478, 527, 482, 536, 535, 529, 485, 525, 512, 519, 485, 490, 488, 527, 486, 516, 498, 494, 537, 528, 553, 485, 484, 499, 514, 533, 522, 503, 536, 493, 489, 515, 517, 525, 537, 518, 499, 543, 500, 528, 468, 494, 543, 475, 529, 476, 473, 513, 507, 519, 520, 563, 482, 506, 513, 467, 533, 494, 516, 534, 535, 526, 507, 518, 502, 548, 526, 489, 485, 536, 527, 477, 530, 531, 519, 537, 493, 475, 523, 501, 521, 581, 550, 493, 524, 518, 480, 508, 486, 493, 500, 518, 535, 537, 503, 519, 507, 482, 503, 497, 506, 548, 528, 521, 525, 471, 533] payload_rows_per_dst=[10922, 11102, 10884, 10908, 10844, 10976, 10950, 10964] dup_rows_per_dst=[5446, 5294, 5424, 5508, 5328, 5604, 5416, 5502] combine_rows_per_src=[10922, 11102, 10884, 10908, 10844, 10976, 10950, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76520 dup_rows=43546 dup_rows_remote=37792 dedup_ratio=0.332230 router_tokens_per_expert=[460, 488, 537, 515, 484, 530, 506, 519, 531, 525, 487, 550, 516, 530, 502, 513, 552, 512, 519, 558, 534, 532, 460, 487, 481, 527, 506, 529, 474, 513, 474, 534, 494, 511, 493, 505, 464, 504, 527, 511, 507, 547, 490, 513, 535, 502, 520, 488, 488, 547, 538, 528, 456, 518, 489, 509, 522, 510, 512, 552, 510, 489, 463, 490, 505, 503, 472, 496, 492, 505, 498, 505, 510, 510, 514, 492, 509, 542, 508, 504, 511, 490, 480, 546, 509, 525, 544, 505, 526, 498, 541, 492, 495, 517, 532, 544, 508, 497, 519, 569, 524, 511, 528, 498, 519, 525, 530, 522, 518, 462, 505, 524, 532, 498, 538, 494, 518, 509, 522, 524, 514, 536, 585, 512, 531, 509, 505, 513] payload_rows_per_dst=[10856, 10964, 10924, 10870, 10858, 11078, 10970, 11006] dup_rows_per_dst=[5530, 5420, 5298, 5372, 5272, 5430, 5470, 5754] combine_rows_per_src=[10856, 10964, 10924, 10870, 10858, 11078, 10970, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87712 payload_rows_remote=76714 dup_rows=43360 dup_rows_remote=38078 dedup_ratio=0.330811 router_tokens_per_expert=[519, 500, 528, 514, 509, 523, 494, 527, 485, 536, 499, 523, 498, 533, 494, 536, 510, 546, 504, 511, 493, 558, 513, 526, 506, 483, 515, 457, 494, 510, 524, 490, 498, 548, 515, 511, 531, 514, 527, 477, 481, 503, 481, 517, 545, 476, 531, 511, 514, 478, 521, 535, 551, 514, 493, 505, 485, 534, 498, 513, 513, 546, 503, 484, 524, 507, 482, 507, 457, 529, 480, 506, 500, 511, 512, 491, 534, 520, 483, 541, 518, 563, 481, 497, 503, 506, 549, 491, 466, 485, 529, 521, 568, 547, 518, 494, 526, 502, 525, 514, 475, 532, 507, 564, 501, 493, 493, 557, 513, 531, 488, 519, 485, 533, 566, 520, 508, 533, 500, 544, 504, 489, 551, 525, 487, 518, 488, 514] payload_rows_per_dst=[11002, 10998, 10922, 11030, 10862, 11010, 10890, 10998] dup_rows_per_dst=[5434, 5282, 5410, 5344, 5306, 5462, 5590, 5532] combine_rows_per_src=[11002, 10998, 10922, 11030, 10862, 11010, 10890, 10998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87734 payload_rows_remote=76764 dup_rows=43338 dup_rows_remote=37870 dedup_ratio=0.330643 router_tokens_per_expert=[528, 526, 536, 518, 530, 502, 518, 536, 557, 501, 505, 508, 469, 526, 508, 507, 516, 473, 517, 497, 492, 531, 540, 526, 479, 529, 523, 498, 524, 507, 464, 495, 537, 538, 487, 506, 527, 499, 509, 493, 515, 524, 488, 495, 509, 529, 540, 523, 542, 510, 481, 512, 485, 513, 508, 494, 480, 502, 521, 534, 500, 508, 504, 511, 526, 544, 538, 512, 528, 532, 482, 498, 528, 483, 498, 494, 500, 482, 517, 515, 504, 525, 506, 530, 564, 507, 541, 519, 501, 505, 525, 471, 477, 514, 486, 529, 508, 526, 518, 522, 524, 517, 484, 530, 521, 519, 512, 513, 514, 507, 516, 539, 494, 515, 502, 507, 506, 555, 506, 556, 513, 493, 499, 507, 522, 486, 518, 496] payload_rows_per_dst=[10970, 10912, 10970, 10942, 11020, 11038, 10996, 10886] dup_rows_per_dst=[5580, 5310, 5468, 5268, 5334, 5370, 5544, 5464] combine_rows_per_src=[10970, 10912, 10970, 10942, 11020, 11038, 10996, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87402 payload_rows_remote=76650 dup_rows=43670 dup_rows_remote=38340 dedup_ratio=0.333176 router_tokens_per_expert=[538, 505, 494, 561, 485, 492, 491, 534, 488, 549, 513, 522, 520, 518, 489, 508, 510, 504, 514, 510, 494, 516, 521, 524, 523, 526, 505, 516, 514, 514, 483, 523, 503, 527, 536, 478, 511, 547, 503, 539, 529, 500, 505, 500, 547, 504, 521, 533, 490, 509, 556, 514, 507, 489, 491, 510, 493, 476, 523, 526, 481, 479, 496, 501, 510, 518, 517, 488, 542, 535, 468, 510, 541, 514, 510, 526, 550, 527, 534, 573, 475, 489, 491, 522, 472, 493, 516, 541, 516, 495, 509, 483, 527, 526, 502, 497, 543, 544, 512, 543, 505, 503, 521, 510, 527, 550, 529, 486, 489, 535, 523, 495, 533, 503, 512, 523, 507, 487, 518, 511, 494, 491, 498, 532, 465, 491, 515, 496] payload_rows_per_dst=[10960, 10976, 10984, 10752, 11106, 10776, 10984, 10864] dup_rows_per_dst=[5454, 5418, 5582, 5330, 5620, 5332, 5646, 5288] combine_rows_per_src=[10960, 10976, 10984, 10752, 11106, 10776, 10984, 10864] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87504 payload_rows_remote=76552 dup_rows=43568 dup_rows_remote=38226 dedup_ratio=0.332397 router_tokens_per_expert=[500, 487, 531, 527, 530, 506, 502, 481, 525, 507, 507, 490, 522, 468, 539, 507, 516, 520, 511, 479, 477, 551, 477, 537, 494, 511, 517, 500, 533, 490, 520, 519, 510, 501, 504, 501, 500, 522, 511, 500, 506, 525, 509, 497, 531, 506, 521, 534, 509, 517, 498, 508, 506, 529, 512, 513, 480, 493, 496, 503, 475, 541, 564, 446, 505, 503, 537, 518, 559, 529, 505, 524, 530, 529, 527, 507, 523, 478, 526, 556, 497, 512, 522, 514, 506, 487, 521, 467, 507, 509, 487, 498, 513, 532, 534, 541, 520, 505, 556, 515, 507, 553, 505, 529, 464, 547, 515, 541, 494, 499, 491, 503, 535, 530, 524, 544, 495, 499, 512, 514, 495, 510, 499, 528, 516, 546, 506, 487] payload_rows_per_dst=[10946, 11004, 10924, 10826, 10994, 10952, 10954, 10904] dup_rows_per_dst=[5312, 5300, 5432, 5354, 5718, 5342, 5534, 5576] combine_rows_per_src=[10946, 11004, 10924, 10826, 10994, 10952, 10954, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627507200 dispatch_meta_bytes=458824 combine_payload_bytes=627507200 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627966024 ingress_bytes=929497088 link_bytes=929497088 token_bytes=1255473224 weight_movement_bytes=301989888 total_nvlink_bytes=1557463112 weight_to_token_ratio=0.240539 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.178733144e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87584 payload_rows_remote=76722 dup_rows=43488 dup_rows_remote=38096 dedup_ratio=0.331787 router_tokens_per_expert=[500, 537, 495, 483, 555, 467, 513, 532, 491, 516, 499, 554, 497, 527, 527, 506, 502, 509, 507, 535, 516, 529, 521, 476, 523, 501, 482, 504, 525, 512, 514, 509, 503, 500, 461, 524, 480, 513, 531, 511, 527, 511, 508, 493, 473, 511, 526, 521, 508, 536, 506, 535, 494, 509, 534, 508, 484, 522, 520, 497, 524, 497, 478, 519, 521, 519, 550, 528, 461, 480, 497, 516, 498, 518, 513, 471, 509, 513, 522, 511, 498, 536, 472, 504, 538, 521, 533, 533, 550, 549, 521, 524, 518, 519, 543, 502, 523, 528, 486, 513, 523, 535, 547, 525, 524, 517, 519, 519, 489, 509, 544, 526, 499, 480, 519, 515, 484, 524, 502, 482, 525, 478, 531, 469, 525, 509, 534, 517] payload_rows_per_dst=[10932, 10960, 10848, 10870, 10862, 11134, 11098, 10880] dup_rows_per_dst=[5466, 5370, 5338, 5472, 5392, 5588, 5556, 5306] combine_rows_per_src=[10932, 10960, 10848, 10870, 10862, 11134, 11098, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=457248 combine_payload_bytes=626851840 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627309088 ingress_bytes=777846784 link_bytes=777846784 token_bytes=1254160928 weight_movement_bytes=150994944 total_nvlink_bytes=1405155872 weight_to_token_ratio=0.120395 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864192123e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87826 payload_rows_remote=76888 dup_rows=43246 dup_rows_remote=37922 dedup_ratio=0.329941 router_tokens_per_expert=[528, 532, 512, 501, 524, 489, 500, 489, 478, 494, 514, 526, 537, 501, 480, 501, 521, 530, 484, 532, 490, 505, 516, 505, 500, 539, 491, 572, 499, 497, 513, 520, 519, 523, 501, 519, 521, 473, 530, 494, 528, 477, 514, 506, 509, 467, 533, 503, 505, 528, 510, 494, 552, 492, 490, 486, 494, 533, 497, 546, 524, 487, 524, 526, 529, 523, 483, 477, 528, 504, 519, 511, 473, 503, 481, 496, 572, 509, 549, 544, 537, 479, 530, 501, 515, 485, 503, 544, 509, 524, 577, 525, 488, 505, 516, 544, 564, 529, 518, 508, 471, 525, 498, 475, 503, 487, 496, 516, 505, 535, 527, 530, 530, 504, 528, 493, 549, 494, 528, 485, 497, 527, 547, 525, 519, 489, 506, 520] payload_rows_per_dst=[10938, 11134, 11040, 11356, 11084, 10756, 10668, 10850] dup_rows_per_dst=[5324, 5628, 5624, 5870, 5472, 5224, 4806, 5298] combine_rows_per_src=[10938, 11134, 11040, 11356, 11084, 10756, 10668, 10850] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628441088 dispatch_meta_bytes=459168 combine_payload_bytes=628441088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628900256 ingress_bytes=703938560 link_bytes=703938560 token_bytes=1257341344 weight_movement_bytes=75497472 total_nvlink_bytes=1332838816 weight_to_token_ratio=0.060045 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926931809e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628850688 dispatch_meta_bytes=458536 combine_payload_bytes=628850688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629309224 ingress_bytes=704348160 link_bytes=704348160 token_bytes=1258159912 weight_movement_bytes=75497472 total_nvlink_bytes=1333657384 weight_to_token_ratio=0.060006 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.932126114e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=459960 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628376760 ingress_bytes=703414272 link_bytes=703414272 token_bytes=1256293560 weight_movement_bytes=75497472 total_nvlink_bytes=1331791032 weight_to_token_ratio=0.060095 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920283099e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627113984 dispatch_meta_bytes=459112 combine_payload_bytes=627113984 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627573096 ingress_bytes=853606400 link_bytes=853606400 token_bytes=1254687080 weight_movement_bytes=226492416 total_nvlink_bytes=1481179496 weight_to_token_ratio=0.180517 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082493069e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628506624 dispatch_meta_bytes=459272 combine_payload_bytes=628506624 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628965896 ingress_bytes=704004096 link_bytes=704004096 token_bytes=1257472520 weight_movement_bytes=75497472 total_nvlink_bytes=1332969992 weight_to_token_ratio=0.060039 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927762898e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=629866496 dispatch_meta_bytes=459240 combine_payload_bytes=629866496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630325736 ingress_bytes=705363968 link_bytes=705363968 token_bytes=1260192232 weight_movement_bytes=75497472 total_nvlink_bytes=1335689704 weight_to_token_ratio=0.059909 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.945007989e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87232 payload_rows_remote=76328 dup_rows=43840 dup_rows_remote=38362 dedup_ratio=0.334473 router_tokens_per_expert=[517, 490, 521, 530, 492, 550, 475, 512, 528, 505, 452, 508, 506, 505, 487, 499, 517, 535, 512, 559, 489, 496, 540, 492, 535, 504, 506, 515, 449, 494, 536, 500, 524, 504, 482, 533, 518, 514, 498, 495, 522, 490, 551, 523, 545, 514, 502, 548, 525, 488, 520, 507, 522, 502, 519, 525, 527, 552, 488, 478, 475, 501, 509, 481, 509, 520, 558, 514, 530, 507, 539, 481, 526, 510, 544, 516, 546, 544, 471, 501, 501, 528, 509, 506, 520, 481, 487, 487, 498, 537, 503, 526, 519, 517, 491, 504, 508, 516, 490, 487, 529, 512, 557, 502, 527, 539, 501, 502, 479, 532, 506, 504, 541, 539, 528, 541, 511, 503, 530, 505, 511, 511, 492, 497, 516, 527, 557, 468] payload_rows_per_dst=[10738, 10938, 10960, 10910, 10938, 10832, 10904, 11012] dup_rows_per_dst=[5416, 5420, 5566, 5328, 5694, 5396, 5478, 5542] combine_rows_per_src=[10738, 10938, 10960, 10910, 10938, 10832, 10904, 11012] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76684 dup_rows=43584 dup_rows_remote=38314 dedup_ratio=0.332520 router_tokens_per_expert=[515, 499, 488, 528, 531, 528, 531, 476, 535, 516, 508, 525, 498, 510, 530, 521, 535, 496, 496, 517, 527, 521, 467, 491, 518, 465, 514, 516, 525, 486, 483, 537, 481, 539, 504, 524, 536, 479, 531, 521, 514, 469, 524, 523, 542, 494, 533, 524, 493, 502, 480, 491, 531, 534, 532, 553, 495, 538, 524, 496, 520, 527, 533, 537, 511, 497, 487, 524, 499, 505, 494, 554, 497, 535, 517, 521, 537, 514, 501, 542, 523, 511, 526, 525, 512, 517, 529, 522, 509, 507, 491, 506, 480, 479, 545, 489, 538, 508, 498, 500, 500, 533, 552, 479, 549, 507, 528, 515, 503, 502, 530, 497, 489, 494, 474, 472, 495, 541, 476, 522, 510, 526, 499, 516, 497, 503, 501, 519] payload_rows_per_dst=[10906, 10872, 10982, 11006, 10782, 10966, 11170, 10804] dup_rows_per_dst=[5572, 5316, 5348, 5638, 5170, 5376, 5894, 5270] combine_rows_per_src=[10906, 10872, 10982, 11006, 10782, 10966, 11170, 10804] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87296 payload_rows_remote=76310 dup_rows=43776 dup_rows_remote=38254 dedup_ratio=0.333984 router_tokens_per_expert=[515, 542, 498, 507, 510, 514, 493, 528, 534, 501, 505, 504, 504, 517, 461, 520, 509, 521, 517, 509, 522, 538, 516, 512, 519, 523, 490, 515, 514, 520, 501, 528, 503, 539, 512, 486, 506, 488, 502, 550, 497, 496, 519, 487, 536, 478, 528, 498, 535, 486, 488, 535, 507, 486, 494, 541, 476, 558, 490, 491, 477, 541, 490, 483, 526, 511, 521, 557, 505, 524, 517, 507, 541, 484, 521, 457, 545, 496, 509, 483, 515, 536, 541, 505, 548, 506, 553, 488, 498, 560, 537, 524, 525, 533, 509, 501, 543, 535, 500, 514, 496, 512, 500, 506, 516, 535, 471, 481, 497, 494, 488, 491, 485, 500, 488, 514, 519, 535, 567, 542, 521, 494, 480, 484, 517, 520, 546, 552] payload_rows_per_dst=[10954, 10986, 10858, 10828, 10930, 10980, 10826, 10934] dup_rows_per_dst=[5352, 5522, 5392, 5328, 5478, 5778, 5332, 5594] combine_rows_per_src=[10954, 10986, 10858, 10828, 10930, 10980, 10826, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87558 payload_rows_remote=76630 dup_rows=43514 dup_rows_remote=37942 dedup_ratio=0.331985 router_tokens_per_expert=[522, 459, 513, 527, 462, 502, 502, 526, 513, 517, 509, 531, 496, 518, 488, 518, 515, 551, 542, 519, 524, 507, 525, 484, 503, 532, 537, 521, 513, 492, 515, 527, 512, 514, 560, 539, 510, 518, 519, 530, 480, 533, 529, 483, 529, 509, 473, 512, 519, 506, 534, 568, 544, 480, 516, 491, 493, 483, 510, 507, 497, 523, 522, 542, 520, 498, 537, 496, 511, 539, 537, 504, 464, 492, 546, 488, 491, 531, 515, 479, 550, 530, 543, 523, 502, 544, 534, 519, 490, 473, 474, 536, 511, 517, 489, 500, 495, 506, 500, 509, 556, 475, 499, 497, 530, 495, 498, 508, 532, 468, 486, 538, 491, 535, 536, 454, 487, 549, 497, 536, 488, 516, 510, 524, 475, 564, 506, 498] payload_rows_per_dst=[10908, 11050, 10928, 10870, 10936, 11058, 10926, 10882] dup_rows_per_dst=[5298, 5564, 5572, 5600, 5360, 5412, 5258, 5450] combine_rows_per_src=[10908, 11050, 10928, 10870, 10936, 11058, 10926, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87414 payload_rows_remote=76432 dup_rows=43658 dup_rows_remote=38208 dedup_ratio=0.333084 router_tokens_per_expert=[530, 556, 517, 486, 532, 486, 495, 538, 499, 500, 479, 529, 513, 508, 515, 493, 528, 537, 528, 520, 553, 528, 473, 508, 549, 507, 533, 515, 545, 505, 487, 507, 543, 483, 498, 490, 512, 530, 505, 519, 496, 531, 537, 487, 445, 555, 482, 511, 512, 497, 480, 495, 537, 500, 504, 503, 526, 537, 515, 516, 505, 542, 509, 538, 505, 504, 485, 533, 516, 529, 489, 511, 498, 530, 543, 534, 574, 505, 476, 513, 555, 491, 519, 530, 567, 552, 548, 530, 508, 541, 478, 462, 460, 523, 495, 502, 504, 481, 516, 493, 474, 549, 493, 475, 501, 470, 491, 486, 500, 509, 532, 503, 522, 507, 548, 520, 522, 549, 494, 517, 510, 479, 504, 504, 513, 486, 544, 495] payload_rows_per_dst=[10916, 11066, 10850, 10982, 10968, 10976, 10694, 10962] dup_rows_per_dst=[5436, 5580, 5398, 5450, 5522, 5546, 5260, 5466] combine_rows_per_src=[10916, 11066, 10850, 10982, 10968, 10976, 10694, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87628 payload_rows_remote=76620 dup_rows=43444 dup_rows_remote=37952 dedup_ratio=0.331451 router_tokens_per_expert=[501, 492, 522, 518, 463, 522, 536, 517, 511, 504, 500, 541, 492, 479, 505, 494, 539, 475, 500, 485, 502, 516, 507, 531, 534, 533, 495, 508, 516, 492, 519, 520, 524, 537, 499, 503, 491, 492, 517, 498, 484, 509, 513, 501, 549, 470, 552, 528, 484, 487, 512, 539, 533, 517, 533, 511, 506, 493, 518, 540, 501, 548, 515, 526, 511, 532, 506, 493, 514, 515, 504, 541, 479, 568, 507, 532, 527, 505, 508, 508, 544, 521, 493, 535, 488, 516, 508, 478, 470, 509, 515, 487, 509, 516, 497, 495, 512, 524, 504, 488, 484, 555, 506, 496, 496, 530, 526, 532, 520, 533, 499, 510, 502, 501, 474, 547, 516, 585, 500, 504, 512, 556, 521, 486, 515, 525, 523, 524] payload_rows_per_dst=[10820, 10912, 10936, 11008, 11008, 10888, 10968, 11088] dup_rows_per_dst=[5374, 5432, 5398, 5518, 5492, 5274, 5462, 5494] combine_rows_per_src=[10820, 10912, 10936, 11008, 11008, 10888, 10968, 11088] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625278976 dispatch_meta_bytes=458760 combine_payload_bytes=625278976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625737736 ingress_bytes=700776448 link_bytes=700776448 token_bytes=1251016712 weight_movement_bytes=75497472 total_nvlink_bytes=1326514184 weight_to_token_ratio=0.060349 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.886831778e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87524 payload_rows_remote=76690 dup_rows=43548 dup_rows_remote=38224 dedup_ratio=0.332245 router_tokens_per_expert=[513, 519, 510, 513, 539, 492, 519, 529, 496, 558, 520, 551, 501, 494, 488, 484, 491, 529, 542, 494, 475, 528, 507, 542, 525, 503, 568, 535, 510, 522, 509, 494, 501, 520, 530, 494, 469, 532, 478, 463, 504, 499, 537, 485, 528, 498, 524, 491, 499, 535, 482, 505, 487, 481, 477, 523, 501, 540, 497, 557, 517, 541, 506, 483, 517, 505, 506, 536, 480, 524, 509, 531, 515, 537, 472, 530, 514, 529, 541, 510, 528, 472, 514, 519, 504, 489, 505, 475, 499, 498, 527, 465, 478, 546, 511, 549, 502, 507, 504, 526, 537, 516, 559, 526, 487, 541, 511, 524, 463, 522, 500, 543, 491, 494, 527, 494, 562, 558, 505, 518, 501, 541, 518, 533, 486, 518, 511, 492] payload_rows_per_dst=[10978, 10946, 10882, 10864, 10968, 10834, 11056, 10996] dup_rows_per_dst=[5474, 5602, 5224, 5398, 5544, 5324, 5480, 5502] combine_rows_per_src=[10978, 10946, 10882, 10864, 10968, 10834, 11056, 10996] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628195328 dispatch_meta_bytes=459992 combine_payload_bytes=628195328 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628655320 ingress_bytes=779190272 link_bytes=779190272 token_bytes=1256850648 weight_movement_bytes=150994944 total_nvlink_bytes=1407845592 weight_to_token_ratio=0.120138 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.881229442e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87548 payload_rows_remote=76364 dup_rows=43524 dup_rows_remote=37778 dedup_ratio=0.332062 router_tokens_per_expert=[500, 504, 506, 498, 527, 499, 530, 515, 499, 545, 528, 506, 526, 482, 524, 514, 513, 514, 518, 496, 501, 521, 494, 452, 489, 523, 519, 510, 524, 529, 489, 554, 525, 531, 534, 528, 511, 527, 533, 495, 497, 516, 508, 552, 527, 525, 592, 537, 521, 517, 524, 531, 553, 509, 521, 478, 525, 498, 511, 516, 486, 482, 526, 474, 517, 515, 473, 548, 480, 527, 512, 520, 517, 478, 517, 491, 491, 498, 527, 530, 520, 517, 492, 497, 476, 514, 481, 551, 499, 503, 517, 503, 521, 486, 514, 492, 501, 501, 507, 545, 502, 508, 504, 500, 517, 514, 474, 552, 486, 559, 528, 484, 544, 505, 496, 535, 492, 527, 531, 517, 515, 531, 508, 521, 437, 487, 496, 529] payload_rows_per_dst=[11184, 10690, 11062, 10982, 10922, 10908, 11068, 10732] dup_rows_per_dst=[5746, 5176, 5716, 5362, 5360, 5524, 5296, 5344] combine_rows_per_src=[11184, 10690, 11062, 10982, 10922, 10908, 11068, 10732] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625131520 dispatch_meta_bytes=458256 combine_payload_bytes=625131520 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625589776 ingress_bytes=776126464 link_bytes=776126464 token_bytes=1250721296 weight_movement_bytes=150994944 total_nvlink_bytes=1401716240 weight_to_token_ratio=0.120726 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.842376044e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627752960 dispatch_meta_bytes=458288 combine_payload_bytes=627752960 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=628211248 ingress_bytes=929742848 link_bytes=929742848 token_bytes=1255964208 weight_movement_bytes=301989888 total_nvlink_bytes=1557954096 weight_to_token_ratio=0.240445 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.179044802e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626130944 dispatch_meta_bytes=458560 combine_payload_bytes=626130944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626589504 ingress_bytes=701628416 link_bytes=701628416 token_bytes=1252720448 weight_movement_bytes=75497472 total_nvlink_bytes=1328217920 weight_to_token_ratio=0.060267 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897635932e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627671040 dispatch_meta_bytes=458288 combine_payload_bytes=627671040 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628129328 ingress_bytes=778665984 link_bytes=778665984 token_bytes=1255800368 weight_movement_bytes=150994944 total_nvlink_bytes=1406795312 weight_to_token_ratio=0.120238 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874580732e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628244480 dispatch_meta_bytes=459656 combine_payload_bytes=628244480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628704136 ingress_bytes=703741952 link_bytes=703741952 token_bytes=1256948616 weight_movement_bytes=75497472 total_nvlink_bytes=1332446088 weight_to_token_ratio=0.060064 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924438543e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625573888 dispatch_meta_bytes=456568 combine_payload_bytes=625573888 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626030456 ingress_bytes=701071360 link_bytes=701071360 token_bytes=1251604344 weight_movement_bytes=75497472 total_nvlink_bytes=1327101816 weight_to_token_ratio=0.060321 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890571677e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87650 payload_rows_remote=76730 dup_rows=43422 dup_rows_remote=37866 dedup_ratio=0.331284 router_tokens_per_expert=[503, 504, 524, 515, 483, 548, 499, 493, 510, 573, 478, 554, 514, 507, 565, 522, 510, 490, 528, 499, 512, 511, 508, 485, 539, 504, 513, 519, 546, 503, 565, 515, 491, 498, 513, 490, 515, 511, 498, 482, 487, 513, 489, 507, 516, 536, 507, 544, 521, 494, 483, 527, 522, 484, 505, 475, 553, 482, 475, 504, 500, 548, 544, 515, 512, 535, 495, 512, 523, 494, 504, 497, 513, 497, 517, 505, 492, 506, 526, 531, 480, 513, 520, 520, 550, 500, 502, 517, 521, 516, 500, 529, 498, 513, 535, 500, 527, 510, 530, 519, 522, 537, 500, 510, 535, 488, 472, 464, 535, 547, 506, 536, 532, 474, 493, 549, 491, 523, 501, 523, 518, 493, 510, 517, 518, 482, 527, 506] payload_rows_per_dst=[11062, 10976, 10914, 10946, 10918, 10990, 10920, 10924] dup_rows_per_dst=[5522, 5518, 5280, 5318, 5400, 5438, 5556, 5390] combine_rows_per_src=[11062, 10976, 10914, 10946, 10918, 10990, 10920, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87170 payload_rows_remote=76218 dup_rows=43902 dup_rows_remote=38236 dedup_ratio=0.334946 router_tokens_per_expert=[541, 548, 520, 494, 534, 515, 485, 488, 484, 511, 541, 501, 485, 557, 563, 523, 526, 491, 498, 563, 518, 504, 509, 551, 561, 529, 532, 533, 471, 564, 476, 481, 534, 525, 502, 542, 503, 519, 568, 512, 508, 551, 519, 480, 528, 480, 494, 520, 508, 490, 503, 519, 521, 513, 460, 496, 505, 464, 529, 476, 527, 503, 498, 484, 469, 490, 484, 512, 523, 516, 505, 509, 531, 481, 471, 482, 520, 493, 456, 514, 488, 494, 503, 529, 517, 522, 503, 529, 505, 468, 545, 520, 507, 503, 547, 529, 525, 546, 544, 533, 509, 507, 511, 516, 510, 513, 542, 499, 505, 492, 484, 510, 535, 519, 517, 525, 512, 500, 529, 487, 521, 570, 476, 528, 540, 502, 514, 472] payload_rows_per_dst=[11022, 10840, 10978, 10770, 10700, 10954, 10954, 10952] dup_rows_per_dst=[5486, 5478, 5616, 5440, 5212, 5464, 5540, 5666] combine_rows_per_src=[11022, 10840, 10978, 10770, 10700, 10954, 10954, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87554 payload_rows_remote=76564 dup_rows=43518 dup_rows_remote=38022 dedup_ratio=0.332016 router_tokens_per_expert=[535, 550, 532, 508, 497, 558, 484, 501, 519, 505, 512, 503, 476, 495, 471, 509, 513, 510, 544, 521, 503, 550, 497, 524, 559, 467, 540, 499, 495, 486, 494, 541, 492, 545, 476, 478, 495, 512, 519, 503, 548, 539, 500, 507, 516, 481, 512, 514, 468, 538, 514, 534, 496, 515, 511, 496, 539, 484, 475, 517, 512, 527, 485, 516, 507, 500, 461, 475, 505, 504, 490, 519, 495, 496, 481, 538, 482, 522, 582, 541, 509, 531, 516, 493, 550, 530, 495, 507, 515, 493, 503, 525, 516, 532, 522, 496, 499, 522, 492, 498, 540, 547, 523, 512, 528, 514, 546, 512, 513, 532, 519, 514, 494, 549, 540, 548, 477, 525, 542, 527, 507, 517, 518, 492, 509, 489, 519, 479] payload_rows_per_dst=[10926, 10990, 10892, 10812, 10884, 10924, 11070, 11056] dup_rows_per_dst=[5384, 5496, 5382, 5442, 5312, 5542, 5552, 5408] combine_rows_per_src=[10926, 10990, 10892, 10812, 10884, 10924, 11070, 11056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87414 payload_rows_remote=76454 dup_rows=43658 dup_rows_remote=38142 dedup_ratio=0.333084 router_tokens_per_expert=[519, 516, 530, 513, 493, 498, 525, 500, 493, 541, 483, 509, 519, 533, 509, 487, 491, 472, 518, 484, 510, 494, 526, 494, 506, 485, 488, 511, 472, 495, 485, 482, 503, 516, 498, 535, 521, 531, 505, 507, 512, 516, 452, 562, 537, 530, 500, 513, 497, 550, 529, 491, 536, 513, 543, 512, 481, 537, 522, 510, 531, 505, 500, 499, 527, 533, 530, 500, 496, 504, 517, 551, 513, 517, 515, 554, 496, 489, 501, 511, 530, 486, 525, 465, 532, 480, 508, 495, 541, 497, 556, 484, 545, 517, 537, 518, 501, 501, 520, 542, 473, 526, 533, 534, 525, 502, 491, 513, 556, 563, 499, 538, 526, 517, 527, 493, 461, 499, 552, 487, 512, 523, 512, 496, 533, 508, 504, 524] payload_rows_per_dst=[10898, 10574, 10960, 11044, 11022, 10946, 11018, 10952] dup_rows_per_dst=[5438, 5252, 5516, 5468, 5486, 5486, 5616, 5396] combine_rows_per_src=[10898, 10574, 10960, 11044, 11022, 10946, 11018, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87280 payload_rows_remote=76360 dup_rows=43792 dup_rows_remote=38160 dedup_ratio=0.334106 router_tokens_per_expert=[532, 492, 522, 529, 532, 489, 529, 486, 508, 521, 493, 510, 477, 520, 499, 500, 498, 533, 499, 526, 530, 492, 488, 502, 518, 511, 502, 526, 534, 534, 497, 486, 539, 537, 540, 513, 513, 524, 477, 506, 515, 492, 477, 544, 536, 503, 518, 538, 516, 546, 514, 494, 516, 474, 509, 522, 541, 543, 521, 486, 525, 502, 513, 554, 524, 527, 520, 528, 508, 475, 563, 474, 508, 527, 502, 518, 523, 485, 546, 511, 529, 480, 509, 516, 488, 499, 552, 506, 516, 518, 487, 496, 499, 535, 515, 511, 543, 486, 523, 524, 509, 510, 515, 498, 504, 531, 529, 479, 478, 540, 503, 486, 484, 507, 499, 511, 521, 542, 526, 500, 543, 522, 480, 482, 492, 488, 478, 545] payload_rows_per_dst=[10878, 10892, 11016, 10920, 10986, 10894, 10856, 10838] dup_rows_per_dst=[5400, 5460, 5528, 5632, 5492, 5418, 5460, 5402] combine_rows_per_src=[10878, 10892, 11016, 10920, 10986, 10894, 10856, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87410 payload_rows_remote=76534 dup_rows=43662 dup_rows_remote=38242 dedup_ratio=0.333115 router_tokens_per_expert=[479, 545, 506, 496, 479, 532, 471, 541, 502, 545, 517, 479, 539, 515, 510, 535, 475, 560, 496, 518, 498, 529, 505, 524, 564, 484, 536, 493, 498, 551, 534, 491, 492, 546, 497, 483, 505, 464, 506, 535, 498, 499, 509, 516, 530, 506, 512, 497, 470, 543, 519, 487, 537, 521, 499, 506, 517, 474, 537, 535, 509, 522, 544, 511, 496, 512, 495, 522, 493, 519, 539, 542, 486, 498, 512, 514, 550, 510, 467, 493, 547, 524, 493, 500, 526, 483, 513, 494, 533, 554, 500, 531, 500, 503, 484, 524, 504, 505, 501, 540, 517, 513, 520, 517, 502, 516, 490, 539, 527, 507, 480, 477, 510, 542, 524, 532, 512, 514, 537, 521, 502, 524, 507, 469, 517, 475, 552, 513] payload_rows_per_dst=[10902, 10956, 10916, 10950, 10876, 11048, 10830, 10932] dup_rows_per_dst=[5480, 5556, 5274, 5512, 5420, 5370, 5480, 5570] combine_rows_per_src=[10902, 10956, 10916, 10950, 10876, 11048, 10830, 10932] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87280 payload_rows_remote=76404 dup_rows=43792 dup_rows_remote=38358 dedup_ratio=0.334106 router_tokens_per_expert=[489, 514, 516, 499, 511, 512, 502, 511, 519, 541, 484, 486, 505, 521, 517, 488, 535, 546, 542, 536, 451, 546, 466, 515, 514, 509, 532, 511, 514, 467, 556, 518, 528, 463, 499, 493, 509, 515, 540, 479, 541, 557, 516, 447, 506, 511, 528, 477, 513, 532, 519, 527, 559, 502, 530, 471, 505, 499, 474, 518, 515, 508, 523, 521, 501, 545, 518, 555, 507, 469, 500, 513, 521, 515, 509, 503, 493, 517, 506, 521, 500, 536, 555, 512, 533, 495, 523, 492, 488, 498, 496, 481, 507, 522, 505, 512, 468, 570, 538, 496, 510, 512, 517, 472, 563, 516, 542, 544, 477, 525, 525, 516, 497, 534, 500, 517, 512, 536, 478, 516, 528, 516, 494, 509, 505, 540, 535, 482] payload_rows_per_dst=[10804, 10986, 10812, 11020, 10908, 10876, 10962, 10912] dup_rows_per_dst=[5426, 5530, 5406, 5412, 5478, 5434, 5620, 5486] combine_rows_per_src=[10804, 10986, 10812, 11020, 10908, 10876, 10962, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87722 payload_rows_remote=76810 dup_rows=43350 dup_rows_remote=37818 dedup_ratio=0.330734 router_tokens_per_expert=[548, 519, 480, 493, 532, 510, 543, 521, 517, 487, 526, 486, 521, 499, 518, 522, 530, 495, 527, 537, 506, 505, 482, 541, 529, 507, 509, 511, 499, 531, 525, 550, 513, 542, 501, 521, 557, 499, 547, 520, 536, 539, 485, 479, 526, 539, 477, 510, 517, 560, 486, 519, 496, 534, 533, 504, 516, 499, 521, 513, 477, 483, 518, 517, 469, 502, 505, 557, 537, 513, 479, 493, 520, 534, 524, 516, 475, 522, 526, 491, 477, 493, 546, 507, 513, 497, 526, 497, 477, 493, 528, 483, 473, 528, 560, 525, 519, 521, 444, 526, 520, 501, 500, 512, 503, 539, 504, 512, 517, 471, 523, 554, 524, 517, 474, 530, 521, 497, 500, 508, 525, 482, 491, 479, 500, 511, 537, 498] payload_rows_per_dst=[10912, 10952, 10964, 11018, 11190, 11062, 10776, 10848] dup_rows_per_dst=[5532, 5616, 5618, 5368, 5788, 5226, 4862, 5340] combine_rows_per_src=[10912, 10952, 10964, 11018, 11190, 11062, 10776, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628572160 dispatch_meta_bytes=458384 combine_payload_bytes=628572160 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=629030544 ingress_bytes=855064576 link_bytes=855064576 token_bytes=1257602704 weight_movement_bytes=226492416 total_nvlink_bytes=1484095120 weight_to_token_ratio=0.180099 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.084342242e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=624377856 dispatch_meta_bytes=457816 combine_payload_bytes=624377856 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624835672 ingress_bytes=699875328 link_bytes=699875328 token_bytes=1249213528 weight_movement_bytes=75497472 total_nvlink_bytes=1324711000 weight_to_token_ratio=0.060436 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.875404308e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627212288 dispatch_meta_bytes=458344 combine_payload_bytes=627212288 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627670632 ingress_bytes=853704704 link_bytes=853704704 token_bytes=1254882920 weight_movement_bytes=226492416 total_nvlink_bytes=1481375336 weight_to_token_ratio=0.180489 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082617733e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626311168 dispatch_meta_bytes=458384 combine_payload_bytes=626311168 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626769552 ingress_bytes=701808640 link_bytes=701808640 token_bytes=1253080720 weight_movement_bytes=75497472 total_nvlink_bytes=1328578192 weight_to_token_ratio=0.060249 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899921426e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626966528 dispatch_meta_bytes=459104 combine_payload_bytes=626966528 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627425632 ingress_bytes=702464000 link_bytes=702464000 token_bytes=1254392160 weight_movement_bytes=75497472 total_nvlink_bytes=1329889632 weight_to_token_ratio=0.060186 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908232313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625541120 dispatch_meta_bytes=458080 combine_payload_bytes=625541120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625999200 ingress_bytes=701038592 link_bytes=701038592 token_bytes=1251540320 weight_movement_bytes=75497472 total_nvlink_bytes=1327037792 weight_to_token_ratio=0.060324 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890156133e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625901568 dispatch_meta_bytes=459048 combine_payload_bytes=625901568 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626360616 ingress_bytes=701399040 link_bytes=701399040 token_bytes=1252262184 weight_movement_bytes=75497472 total_nvlink_bytes=1327759656 weight_to_token_ratio=0.060289 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894727121e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=629227520 dispatch_meta_bytes=458512 combine_payload_bytes=629227520 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=629686032 ingress_bytes=855719936 link_bytes=855719936 token_bytes=1258913552 weight_movement_bytes=226492416 total_nvlink_bytes=1485405968 weight_to_token_ratio=0.179911 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085173331e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87388 payload_rows_remote=76302 dup_rows=43684 dup_rows_remote=38114 dedup_ratio=0.333282 router_tokens_per_expert=[502, 497, 522, 532, 520, 492, 515, 515, 524, 526, 484, 494, 498, 548, 510, 550, 465, 537, 517, 528, 550, 520, 523, 525, 519, 482, 542, 543, 500, 513, 519, 478, 541, 509, 529, 478, 541, 499, 466, 520, 482, 477, 520, 491, 501, 543, 498, 488, 495, 490, 539, 508, 507, 515, 511, 501, 494, 522, 507, 507, 518, 496, 516, 531, 523, 494, 550, 523, 503, 472, 530, 491, 502, 499, 522, 505, 525, 513, 505, 509, 492, 484, 524, 529, 518, 498, 555, 523, 533, 549, 490, 484, 507, 490, 502, 519, 478, 550, 505, 523, 536, 515, 525, 493, 466, 542, 527, 549, 528, 531, 542, 518, 481, 488, 527, 492, 487, 538, 499, 501, 516, 516, 492, 530, 522, 504, 516, 506] payload_rows_per_dst=[10894, 10986, 10882, 10912, 10956, 10876, 11086, 10796] dup_rows_per_dst=[5564, 5536, 5284, 5402, 5376, 5518, 5570, 5434] combine_rows_per_src=[10894, 10986, 10882, 10912, 10956, 10876, 11086, 10796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87340 payload_rows_remote=76350 dup_rows=43732 dup_rows_remote=38312 dedup_ratio=0.333649 router_tokens_per_expert=[499, 468, 516, 528, 539, 496, 495, 525, 495, 547, 489, 455, 554, 527, 501, 501, 499, 532, 517, 522, 502, 510, 443, 520, 545, 518, 496, 493, 503, 522, 493, 563, 508, 514, 476, 547, 509, 488, 542, 539, 518, 525, 503, 521, 509, 554, 532, 495, 521, 529, 551, 487, 501, 544, 549, 549, 510, 486, 488, 491, 520, 521, 510, 531, 528, 532, 472, 523, 544, 516, 514, 517, 508, 447, 509, 520, 519, 497, 481, 530, 523, 488, 526, 495, 491, 547, 538, 502, 539, 476, 521, 536, 546, 503, 511, 504, 488, 488, 542, 484, 471, 480, 495, 517, 534, 508, 535, 509, 491, 501, 517, 530, 488, 507, 530, 572, 511, 493, 507, 497, 522, 512, 488, 491, 520, 488, 525, 511] payload_rows_per_dst=[10746, 10748, 11042, 10838, 11058, 11004, 10914, 10990] dup_rows_per_dst=[5278, 5392, 5734, 5438, 5656, 5488, 5326, 5420] combine_rows_per_src=[10746, 10748, 11042, 10838, 11058, 11004, 10914, 10990] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87598 payload_rows_remote=76680 dup_rows=43474 dup_rows_remote=37838 dedup_ratio=0.331680 router_tokens_per_expert=[526, 529, 503, 513, 538, 498, 517, 505, 476, 533, 524, 508, 522, 528, 529, 529, 478, 521, 528, 499, 508, 481, 555, 538, 556, 510, 492, 487, 521, 542, 524, 537, 536, 502, 506, 538, 543, 491, 555, 483, 505, 526, 512, 529, 473, 479, 494, 514, 477, 485, 476, 529, 519, 521, 488, 503, 470, 485, 567, 519, 538, 504, 512, 501, 529, 522, 475, 524, 521, 494, 512, 507, 511, 475, 515, 540, 467, 529, 479, 522, 531, 504, 514, 493, 504, 507, 501, 528, 531, 510, 487, 482, 524, 556, 503, 517, 495, 518, 480, 518, 496, 520, 543, 531, 506, 558, 513, 515, 479, 511, 484, 484, 518, 563, 508, 535, 539, 517, 520, 506, 487, 512, 540, 499, 491, 508, 503, 490] payload_rows_per_dst=[11144, 10918, 10952, 10904, 10856, 10876, 10962, 10986] dup_rows_per_dst=[5412, 5636, 5420, 5284, 5388, 5508, 5340, 5486] combine_rows_per_src=[11144, 10918, 10952, 10904, 10856, 10876, 10962, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87634 payload_rows_remote=76664 dup_rows=43438 dup_rows_remote=37888 dedup_ratio=0.331406 router_tokens_per_expert=[534, 534, 502, 523, 514, 511, 462, 506, 535, 505, 524, 520, 506, 497, 516, 540, 489, 527, 490, 508, 541, 526, 472, 538, 526, 524, 505, 494, 514, 511, 514, 525, 500, 468, 520, 489, 543, 486, 544, 488, 505, 547, 491, 530, 547, 551, 498, 553, 511, 518, 496, 538, 539, 511, 496, 510, 528, 486, 542, 501, 536, 522, 490, 504, 498, 490, 512, 483, 541, 508, 487, 510, 510, 518, 475, 529, 523, 538, 520, 517, 526, 525, 477, 529, 525, 517, 489, 537, 527, 522, 533, 500, 560, 533, 473, 542, 512, 522, 517, 545, 484, 504, 520, 491, 513, 511, 529, 457, 473, 512, 556, 511, 501, 552, 523, 470, 455, 484, 489, 520, 482, 516, 526, 518, 487, 487, 477, 497] payload_rows_per_dst=[10946, 10934, 10970, 11046, 10916, 11114, 10912, 10796] dup_rows_per_dst=[5512, 5474, 5550, 5410, 5402, 5516, 5402, 5172] combine_rows_per_src=[10946, 10934, 10970, 11046, 10916, 11114, 10912, 10796] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87674 payload_rows_remote=76610 dup_rows=43398 dup_rows_remote=37998 dedup_ratio=0.331100 router_tokens_per_expert=[492, 508, 521, 518, 484, 527, 495, 491, 510, 473, 541, 464, 526, 507, 543, 551, 478, 523, 500, 449, 484, 510, 522, 502, 528, 501, 513, 535, 516, 537, 520, 486, 489, 506, 532, 485, 558, 499, 542, 501, 501, 509, 518, 509, 473, 453, 530, 507, 532, 529, 527, 524, 499, 473, 509, 533, 547, 513, 525, 472, 497, 514, 570, 538, 503, 516, 520, 512, 508, 535, 517, 502, 519, 529, 513, 505, 497, 533, 502, 521, 488, 479, 517, 528, 514, 481, 530, 543, 514, 492, 516, 498, 483, 525, 523, 496, 532, 485, 514, 524, 525, 510, 519, 515, 516, 532, 538, 482, 491, 557, 528, 519, 491, 517, 521, 517, 495, 514, 497, 525, 517, 492, 515, 494, 531, 501, 568, 526] payload_rows_per_dst=[10942, 10886, 10866, 11094, 11064, 10886, 10942, 10994] dup_rows_per_dst=[5360, 5322, 5358, 5510, 5400, 5368, 5632, 5448] combine_rows_per_src=[10942, 10886, 10866, 11094, 11064, 10886, 10942, 10994] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87544 payload_rows_remote=76684 dup_rows=43528 dup_rows_remote=38040 dedup_ratio=0.332092 router_tokens_per_expert=[532, 479, 528, 511, 500, 536, 494, 541, 519, 518, 514, 472, 522, 506, 506, 519, 506, 539, 488, 542, 518, 518, 494, 516, 519, 517, 501, 510, 492, 512, 514, 508, 529, 518, 480, 504, 539, 552, 537, 527, 522, 545, 495, 501, 532, 499, 498, 526, 527, 499, 532, 493, 539, 507, 515, 486, 494, 488, 520, 516, 521, 519, 494, 524, 518, 526, 534, 495, 504, 483, 496, 490, 484, 503, 545, 532, 531, 531, 516, 521, 494, 509, 516, 505, 495, 520, 513, 486, 507, 481, 527, 518, 516, 495, 504, 494, 516, 476, 549, 504, 500, 509, 487, 486, 506, 488, 524, 532, 536, 498, 540, 492, 545, 489, 498, 511, 540, 491, 484, 540, 509, 498, 519, 511, 483, 539, 517, 561] payload_rows_per_dst=[10944, 10936, 11062, 10860, 10918, 10834, 10974, 11016] dup_rows_per_dst=[5450, 5452, 5546, 5488, 5500, 5326, 5312, 5454] combine_rows_per_src=[10944, 10936, 11062, 10860, 10918, 10834, 10974, 11016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625459200 dispatch_meta_bytes=458648 combine_payload_bytes=625459200 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625917848 ingress_bytes=700956672 link_bytes=700956672 token_bytes=1251377048 weight_movement_bytes=75497472 total_nvlink_bytes=1326874520 weight_to_token_ratio=0.060332 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889117272e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87534 payload_rows_remote=76578 dup_rows=43538 dup_rows_remote=38094 dedup_ratio=0.332169 router_tokens_per_expert=[487, 502, 501, 469, 491, 508, 559, 558, 546, 505, 525, 505, 483, 518, 509, 512, 522, 536, 528, 498, 512, 558, 543, 515, 531, 502, 502, 482, 502, 461, 517, 525, 517, 500, 509, 486, 504, 470, 519, 535, 501, 521, 562, 497, 471, 539, 529, 477, 561, 513, 480, 524, 524, 499, 501, 501, 485, 518, 527, 494, 504, 479, 528, 466, 493, 513, 505, 505, 514, 529, 527, 489, 520, 483, 533, 541, 528, 493, 512, 528, 505, 528, 535, 491, 503, 515, 498, 516, 526, 496, 509, 500, 511, 528, 555, 484, 495, 503, 499, 536, 553, 557, 554, 488, 523, 467, 498, 528, 514, 511, 482, 515, 504, 494, 549, 569, 502, 501, 545, 500, 518, 492, 476, 534, 529, 491, 513, 530] payload_rows_per_dst=[10992, 10908, 10872, 10874, 10900, 10956, 11066, 10966] dup_rows_per_dst=[5364, 5560, 5402, 5334, 5526, 5444, 5380, 5528] combine_rows_per_src=[10992, 10908, 10872, 10874, 10900, 10956, 11066, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76466 dup_rows=43582 dup_rows_remote=38082 dedup_ratio=0.332504 router_tokens_per_expert=[542, 511, 510, 551, 510, 535, 537, 520, 533, 525, 490, 475, 500, 494, 533, 485, 496, 520, 494, 514, 546, 500, 528, 512, 520, 512, 532, 492, 537, 501, 518, 481, 519, 500, 530, 508, 510, 491, 527, 491, 503, 462, 493, 482, 525, 519, 507, 499, 494, 514, 505, 529, 511, 508, 559, 543, 556, 529, 438, 545, 529, 515, 532, 532, 509, 495, 510, 494, 522, 466, 531, 501, 552, 506, 479, 509, 489, 518, 511, 486, 515, 522, 524, 559, 509, 465, 479, 525, 543, 507, 528, 514, 497, 495, 502, 495, 530, 454, 529, 516, 523, 526, 539, 505, 494, 519, 508, 515, 519, 460, 487, 503, 509, 502, 573, 546, 507, 545, 512, 539, 510, 499, 525, 508, 524, 491, 475, 528] payload_rows_per_dst=[11024, 10900, 10808, 11148, 10746, 10932, 10948, 10984] dup_rows_per_dst=[5500, 5484, 5324, 5530, 5410, 5426, 5306, 5602] combine_rows_per_src=[11024, 10900, 10808, 11148, 10746, 10932, 10948, 10984] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625065984 dispatch_meta_bytes=457664 combine_payload_bytes=625065984 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625523648 ingress_bytes=700563456 link_bytes=700563456 token_bytes=1250589632 weight_movement_bytes=75497472 total_nvlink_bytes=1326087104 weight_to_token_ratio=0.060370 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.884130740e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628162560 dispatch_meta_bytes=458072 combine_payload_bytes=628162560 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628620632 ingress_bytes=779157504 link_bytes=779157504 token_bytes=1256783192 weight_movement_bytes=150994944 total_nvlink_bytes=1407778136 weight_to_token_ratio=0.120144 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.880813898e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628031488 dispatch_meta_bytes=458208 combine_payload_bytes=628031488 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628489696 ingress_bytes=703528960 link_bytes=703528960 token_bytes=1256521184 weight_movement_bytes=75497472 total_nvlink_bytes=1332018656 weight_to_token_ratio=0.060085 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921737505e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627589120 dispatch_meta_bytes=458432 combine_payload_bytes=627589120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628047552 ingress_bytes=703086592 link_bytes=703086592 token_bytes=1255636672 weight_movement_bytes=75497472 total_nvlink_bytes=1331134144 weight_to_token_ratio=0.060127 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916127656e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15042734448 prefetch_bytes=905969664 reduce_bytes=830472192 weight_movement_bytes=1736441856 egress_bytes=7524116848 ingress_bytes=9255059456 link_bytes=9255059456 total_nvlink_bytes=16779176304 weight_to_token_ratio=0.115434 prefetch_share_of_ingress=0.097889 dedup_ratio=0.332745 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=11 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042279 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.780598449e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=628195328 dispatch_meta_bytes=458896 combine_payload_bytes=628195328 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628654224 ingress_bytes=779190272 link_bytes=779190272 token_bytes=1256849552 weight_movement_bytes=150994944 total_nvlink_bytes=1407844496 weight_to_token_ratio=0.120138 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.881229442e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15035823512 prefetch_bytes=830472192 reduce_bytes=754974720 weight_movement_bytes=1585446912 egress_bytes=7520662936 ingress_bytes=9100607488 link_bytes=9100607488 total_nvlink_bytes=16621270424 weight_to_token_ratio=0.105445 prefetch_share_of_ingress=0.091255 dedup_ratio=0.333176 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=10 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.617376085e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627326976 dispatch_meta_bytes=458688 combine_payload_bytes=627326976 prefetch_bytes=0 reduce_bytes=0 egress_bytes=627785664 ingress_bytes=627326976 link_bytes=627785664 token_bytes=1255112640 weight_movement_bytes=0 total_nvlink_bytes=1255112640 weight_to_token_ratio=0.000000 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=7.961205895e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15048505480 prefetch_bytes=754974720 reduce_bytes=754974720 weight_movement_bytes=1509949440 egress_bytes=7527004296 ingress_bytes=9031450624 link_bytes=9031450624 total_nvlink_bytes=16558454920 weight_to_token_ratio=0.100339 prefetch_share_of_ingress=0.083594 dedup_ratio=0.332765 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=10 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043811 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.544292220e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=9 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626409472 dispatch_meta_bytes=458192 combine_payload_bytes=626409472 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626867664 ingress_bytes=928399360 link_bytes=928399360 token_bytes=1253277136 weight_movement_bytes=301989888 total_nvlink_bytes=1555267024 weight_to_token_ratio=0.240960 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177341071e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15059713392 prefetch_bytes=754974720 reduce_bytes=830472192 weight_movement_bytes=1585446912 egress_bytes=7532608880 ingress_bytes=9112551424 link_bytes=9112551424 total_nvlink_bytes=16645160304 weight_to_token_ratio=0.105277 prefetch_share_of_ingress=0.082850 dedup_ratio=0.332231 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=11 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042126 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.629998245e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15067545408 prefetch_bytes=679477248 reduce_bytes=1132462080 weight_movement_bytes=1811939328 egress_bytes=7536525120 ingress_bytes=9342959616 link_bytes=9342959616 total_nvlink_bytes=16879484736 weight_to_token_ratio=0.120254 prefetch_share_of_ingress=0.072726 dedup_ratio=0.332109 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=15 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042433 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.873489928e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15064630512 prefetch_bytes=905969664 reduce_bytes=301989888 weight_movement_bytes=1207959552 egress_bytes=7535068400 ingress_bytes=8737521664 link_bytes=8737521664 total_nvlink_bytes=16272590064 weight_to_token_ratio=0.080185 prefetch_share_of_ingress=0.103687 dedup_ratio=0.332455 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=4 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.233672808e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15059715824 prefetch_bytes=603979776 reduce_bytes=754974720 weight_movement_bytes=1358954496 egress_bytes=7532611312 ingress_bytes=8886059008 link_bytes=8886517696 total_nvlink_bytes=16418670320 weight_to_token_ratio=0.090238 prefetch_share_of_ingress=0.067969 dedup_ratio=0.332565 prefetch_slots=16 unused_prefetch_slots=368 slot_utilization=0.041667 reduce_slots=10 remote_demand=8 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=26924 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.391129425e-05 prefetch_bytes_per_gemm_flop=6.382761438e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=9 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15043585592 prefetch_bytes=830472192 reduce_bytes=905969664 weight_movement_bytes=1736441856 egress_bytes=7524542008 ingress_bytes=9255485440 link_bytes=9255485440 total_nvlink_bytes=16780027448 weight_to_token_ratio=0.115427 prefetch_share_of_ingress=0.089728 dedup_ratio=0.332509 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=12 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.781048622e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none + [2026-08-29 23:36:25] iteration 10/ 12 | consumed samples: 80 | elapsed time per iteration (ms): 354.3 | throughput per GPU (TFLOP/s/GPU): 510.3 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.434896E+00 | loss scale: 1.0 | grad norm: 5.455 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87380 payload_rows_remote=76520 dup_rows=43692 dup_rows_remote=38256 dedup_ratio=0.333344 router_tokens_per_expert=[541, 533, 492, 518, 519, 492, 491, 490, 554, 514, 476, 515, 477, 550, 531, 493, 521, 515, 539, 559, 474, 515, 492, 454, 578, 492, 546, 561, 518, 531, 521, 484, 495, 507, 541, 510, 481, 542, 523, 513, 502, 507, 506, 499, 474, 504, 503, 541, 512, 471, 536, 503, 493, 520, 496, 497, 506, 531, 521, 492, 531, 495, 487, 549, 537, 503, 486, 531, 490, 500, 534, 541, 517, 516, 534, 489, 514, 484, 535, 493, 511, 460, 515, 485, 497, 483, 538, 546, 509, 471, 497, 527, 478, 511, 555, 523, 544, 497, 494, 514, 513, 515, 548, 520, 534, 493, 513, 527, 516, 511, 507, 516, 512, 521, 506, 511, 502, 492, 489, 567, 504, 483, 538, 506, 528, 510, 536, 485] payload_rows_per_dst=[10878, 10960, 10860, 10896, 11010, 10888, 11014, 10874] dup_rows_per_dst=[5494, 5640, 5436, 5384, 5398, 5324, 5510, 5506] combine_rows_per_src=[10878, 10960, 10860, 10896, 11010, 10888, 11014, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87420 payload_rows_remote=76522 dup_rows=43652 dup_rows_remote=38244 dedup_ratio=0.333038 router_tokens_per_expert=[507, 498, 560, 520, 534, 510, 527, 487, 533, 550, 512, 553, 521, 511, 494, 526, 506, 516, 482, 504, 562, 526, 543, 511, 476, 510, 497, 512, 510, 517, 478, 503, 477, 482, 499, 493, 505, 487, 511, 487, 492, 512, 506, 505, 574, 546, 527, 500, 510, 522, 503, 499, 503, 491, 511, 497, 553, 549, 472, 492, 483, 504, 530, 521, 515, 504, 516, 530, 512, 512, 504, 502, 530, 533, 501, 543, 491, 523, 503, 507, 545, 508, 504, 482, 535, 515, 533, 521, 502, 536, 490, 467, 546, 506, 519, 545, 495, 524, 523, 518, 525, 531, 489, 501, 499, 498, 548, 532, 499, 471, 492, 456, 535, 488, 527, 516, 524, 520, 469, 513, 537, 503, 523, 477, 532, 522, 523, 507] payload_rows_per_dst=[11070, 10898, 10858, 10980, 11004, 10982, 10772, 10856] dup_rows_per_dst=[5616, 5408, 5348, 5300, 5448, 5526, 5430, 5576] combine_rows_per_src=[11070, 10898, 10858, 10980, 11004, 10982, 10772, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87546 payload_rows_remote=76682 dup_rows=43526 dup_rows_remote=38070 dedup_ratio=0.332077 router_tokens_per_expert=[482, 556, 503, 506, 492, 545, 491, 510, 538, 509, 515, 508, 511, 486, 511, 505, 564, 472, 536, 514, 522, 519, 512, 465, 538, 519, 508, 509, 513, 551, 502, 495, 550, 494, 517, 519, 500, 543, 492, 526, 526, 507, 511, 523, 556, 500, 535, 515, 482, 511, 508, 529, 478, 504, 502, 537, 492, 545, 501, 497, 501, 526, 500, 504, 499, 517, 499, 538, 492, 517, 512, 491, 503, 521, 545, 508, 485, 488, 535, 520, 524, 486, 494, 514, 504, 529, 529, 510, 506, 516, 516, 511, 515, 525, 504, 535, 458, 504, 553, 548, 532, 506, 513, 535, 502, 465, 485, 530, 503, 499, 457, 570, 529, 508, 513, 499, 492, 517, 502, 532, 501, 513, 492, 508, 503, 515, 491, 535] payload_rows_per_dst=[10938, 10900, 11076, 10812, 10912, 11192, 10864, 10852] dup_rows_per_dst=[5398, 5578, 5552, 5422, 5428, 5244, 5456, 5448] combine_rows_per_src=[10938, 10900, 11076, 10812, 10912, 11192, 10864, 10852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87640 payload_rows_remote=76760 dup_rows=43432 dup_rows_remote=38014 dedup_ratio=0.331360 router_tokens_per_expert=[514, 507, 563, 484, 477, 472, 505, 537, 509, 530, 497, 509, 536, 503, 507, 513, 537, 526, 518, 513, 524, 518, 487, 538, 518, 515, 502, 519, 508, 511, 527, 490, 530, 515, 510, 527, 524, 545, 513, 503, 489, 504, 527, 501, 522, 565, 472, 552, 513, 458, 479, 522, 540, 513, 518, 522, 519, 503, 503, 519, 540, 495, 488, 502, 537, 530, 488, 508, 473, 525, 511, 540, 512, 501, 514, 547, 539, 488, 481, 498, 486, 549, 515, 524, 522, 488, 515, 501, 477, 531, 514, 555, 488, 488, 519, 533, 514, 553, 518, 457, 499, 493, 521, 479, 490, 481, 517, 575, 491, 562, 487, 517, 481, 504, 550, 460, 519, 504, 508, 516, 503, 501, 543, 550, 519, 502, 484, 494] payload_rows_per_dst=[10888, 10822, 11058, 11226, 10992, 10912, 10862, 10880] dup_rows_per_dst=[5346, 5148, 5532, 5666, 5374, 5502, 5446, 5418] combine_rows_per_src=[10888, 10822, 11058, 11226, 10992, 10912, 10862, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87732 payload_rows_remote=76692 dup_rows=43340 dup_rows_remote=37782 dedup_ratio=0.330658 router_tokens_per_expert=[529, 502, 569, 504, 533, 493, 526, 517, 507, 552, 486, 480, 532, 496, 492, 528, 502, 495, 493, 510, 490, 501, 463, 531, 519, 486, 499, 468, 526, 562, 510, 537, 521, 479, 476, 499, 519, 493, 506, 468, 456, 512, 508, 532, 506, 544, 511, 499, 530, 467, 510, 520, 528, 483, 510, 515, 534, 519, 502, 517, 500, 572, 513, 544, 511, 497, 521, 509, 466, 525, 495, 505, 521, 502, 517, 527, 477, 532, 499, 523, 490, 533, 496, 488, 505, 554, 511, 491, 515, 565, 500, 547, 511, 537, 526, 530, 516, 506, 516, 519, 504, 507, 505, 548, 516, 544, 540, 544, 539, 525, 546, 528, 534, 524, 518, 498, 520, 523, 509, 474, 480, 511, 536, 467, 493, 526, 502, 461] payload_rows_per_dst=[10990, 10900, 10826, 11010, 10954, 11040, 11098, 10914] dup_rows_per_dst=[5502, 5284, 5232, 5518, 5300, 5558, 5708, 5238] combine_rows_per_src=[10990, 10900, 10826, 11010, 10954, 11040, 11098, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87280 payload_rows_remote=76446 dup_rows=43792 dup_rows_remote=38294 dedup_ratio=0.334106 router_tokens_per_expert=[523, 472, 526, 503, 504, 533, 526, 528, 535, 528, 517, 506, 502, 516, 506, 504, 502, 502, 507, 469, 514, 513, 499, 533, 520, 475, 460, 488, 493, 516, 512, 522, 546, 510, 471, 529, 471, 521, 521, 525, 490, 496, 494, 478, 536, 521, 516, 499, 498, 524, 522, 509, 489, 513, 543, 506, 503, 501, 522, 535, 522, 486, 487, 506, 512, 528, 549, 509, 511, 538, 511, 535, 491, 515, 503, 464, 493, 538, 486, 503, 501, 463, 505, 474, 478, 518, 546, 493, 489, 517, 531, 525, 523, 522, 514, 526, 555, 549, 510, 508, 555, 552, 470, 548, 484, 497, 512, 515, 512, 500, 534, 498, 480, 488, 511, 551, 531, 553, 545, 514, 508, 533, 534, 522, 557, 521, 513, 521] payload_rows_per_dst=[10938, 10870, 10768, 10834, 10850, 10888, 11030, 11102] dup_rows_per_dst=[5520, 5180, 5480, 5498, 5522, 5362, 5568, 5662] combine_rows_per_src=[10938, 10870, 10768, 10834, 10850, 10888, 11030, 11102] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626851840 dispatch_meta_bytes=459104 combine_payload_bytes=626851840 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627310944 ingress_bytes=777846784 link_bytes=777846784 token_bytes=1254162784 weight_movement_bytes=150994944 total_nvlink_bytes=1405157728 weight_to_token_ratio=0.120395 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864192123e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626868224 dispatch_meta_bytes=459064 combine_payload_bytes=626868224 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627327288 ingress_bytes=777863168 link_bytes=777863168 token_bytes=1254195512 weight_movement_bytes=150994944 total_nvlink_bytes=1405190456 weight_to_token_ratio=0.120392 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864399895e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628178944 dispatch_meta_bytes=459008 combine_payload_bytes=628178944 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628637952 ingress_bytes=703676416 link_bytes=703676416 token_bytes=1256816896 weight_movement_bytes=75497472 total_nvlink_bytes=1332314368 weight_to_token_ratio=0.060070 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.923607454e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87526 payload_rows_remote=76588 dup_rows=43546 dup_rows_remote=38040 dedup_ratio=0.332230 router_tokens_per_expert=[514, 501, 495, 480, 507, 486, 537, 496, 530, 499, 468, 521, 509, 505, 515, 507, 488, 518, 501, 497, 552, 526, 497, 541, 558, 520, 517, 519, 511, 507, 497, 525, 549, 519, 511, 559, 531, 525, 508, 532, 522, 531, 546, 489, 485, 515, 538, 517, 491, 554, 505, 539, 487, 511, 542, 504, 482, 503, 506, 475, 522, 503, 514, 498, 508, 500, 471, 533, 484, 481, 483, 519, 523, 550, 496, 525, 554, 555, 518, 522, 469, 510, 532, 525, 511, 559, 503, 515, 520, 489, 537, 487, 507, 517, 507, 504, 532, 473, 495, 530, 489, 476, 487, 485, 505, 498, 517, 501, 493, 509, 500, 526, 497, 514, 539, 525, 505, 537, 487, 522, 467, 509, 540, 535, 507, 506, 503, 556] payload_rows_per_dst=[10822, 11062, 11080, 10968, 10938, 10912, 10810, 10934] dup_rows_per_dst=[5318, 5486, 5674, 5304, 5506, 5472, 5222, 5564] combine_rows_per_src=[10822, 11062, 11080, 10968, 10938, 10912, 10810, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628817920 dispatch_meta_bytes=459096 combine_payload_bytes=628817920 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629277016 ingress_bytes=779812864 link_bytes=779812864 token_bytes=1258094936 weight_movement_bytes=150994944 total_nvlink_bytes=1409089880 weight_to_token_ratio=0.120019 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.889124785e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628260864 dispatch_meta_bytes=457896 combine_payload_bytes=628260864 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628718760 ingress_bytes=703758336 link_bytes=703758336 token_bytes=1256979624 weight_movement_bytes=75497472 total_nvlink_bytes=1332477096 weight_to_token_ratio=0.060063 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924646315e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626245632 dispatch_meta_bytes=458960 combine_payload_bytes=626245632 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626704592 ingress_bytes=701743104 link_bytes=701743104 token_bytes=1252950224 weight_movement_bytes=75497472 total_nvlink_bytes=1328447696 weight_to_token_ratio=0.060256 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899090337e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76672 dup_rows=43506 dup_rows_remote=38046 dedup_ratio=0.331924 router_tokens_per_expert=[494, 509, 502, 506, 500, 530, 472, 524, 526, 502, 500, 504, 506, 513, 514, 562, 512, 517, 533, 527, 526, 562, 549, 539, 504, 551, 521, 517, 526, 527, 540, 517, 496, 473, 509, 545, 513, 549, 528, 518, 526, 523, 489, 496, 489, 507, 490, 495, 499, 538, 493, 526, 500, 491, 496, 501, 504, 533, 502, 522, 504, 518, 508, 492, 499, 556, 517, 502, 505, 505, 522, 555, 529, 470, 498, 515, 526, 504, 498, 530, 540, 515, 538, 521, 460, 477, 502, 509, 514, 547, 484, 523, 500, 541, 472, 492, 493, 544, 483, 438, 511, 477, 494, 552, 504, 488, 527, 533, 516, 501, 497, 519, 533, 511, 518, 540, 525, 536, 517, 483, 532, 513, 515, 495, 509, 464, 516, 481] payload_rows_per_dst=[10894, 11188, 10860, 10950, 10952, 10896, 10952, 10874] dup_rows_per_dst=[5460, 5748, 5432, 5304, 5484, 5374, 5330, 5374] combine_rows_per_src=[10894, 11188, 10860, 10950, 10952, 10896, 10952, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627408896 dispatch_meta_bytes=458512 combine_payload_bytes=627408896 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627867408 ingress_bytes=853901312 link_bytes=853901312 token_bytes=1255276304 weight_movement_bytes=226492416 total_nvlink_bytes=1481768720 weight_to_token_ratio=0.180432 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082867059e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87422 payload_rows_remote=76508 dup_rows=43650 dup_rows_remote=38156 dedup_ratio=0.333023 router_tokens_per_expert=[471, 504, 518, 505, 476, 554, 520, 540, 522, 532, 532, 523, 511, 506, 503, 525, 516, 499, 514, 539, 534, 522, 517, 507, 533, 541, 499, 475, 472, 506, 520, 510, 571, 502, 532, 506, 545, 474, 517, 514, 522, 511, 509, 510, 476, 510, 532, 543, 503, 545, 510, 511, 523, 486, 483, 491, 523, 502, 540, 488, 531, 492, 490, 504, 477, 521, 526, 512, 500, 517, 511, 529, 526, 465, 498, 518, 560, 529, 513, 504, 507, 503, 516, 531, 480, 532, 504, 525, 521, 476, 518, 489, 510, 496, 514, 539, 495, 496, 493, 509, 547, 536, 526, 483, 509, 504, 507, 524, 513, 521, 507, 542, 494, 528, 506, 523, 482, 515, 505, 527, 494, 514, 522, 508, 451, 541, 492, 513] payload_rows_per_dst=[11018, 10914, 11040, 10896, 10840, 10848, 11004, 10862] dup_rows_per_dst=[5466, 5494, 5508, 5348, 5572, 5474, 5420, 5368] combine_rows_per_src=[11018, 10914, 11040, 10896, 10840, 10848, 11004, 10862] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87288 payload_rows_remote=76396 dup_rows=43784 dup_rows_remote=38262 dedup_ratio=0.334045 router_tokens_per_expert=[476, 514, 568, 521, 534, 492, 516, 528, 503, 517, 508, 500, 503, 507, 533, 509, 535, 523, 514, 521, 499, 511, 497, 562, 532, 510, 526, 519, 533, 543, 502, 510, 503, 520, 511, 489, 509, 543, 522, 502, 515, 519, 519, 516, 550, 489, 494, 506, 511, 472, 503, 520, 482, 522, 512, 522, 510, 517, 529, 502, 488, 485, 496, 482, 497, 486, 510, 519, 539, 510, 503, 520, 482, 481, 515, 499, 543, 499, 511, 487, 509, 527, 506, 519, 537, 501, 498, 518, 525, 521, 489, 522, 525, 489, 518, 552, 529, 521, 488, 502, 502, 501, 538, 517, 521, 483, 547, 530, 512, 506, 515, 518, 474, 505, 542, 501, 510, 515, 495, 473, 528, 521, 491, 485, 480, 507, 545, 551] payload_rows_per_dst=[11034, 11060, 10892, 10688, 10856, 10992, 10980, 10786] dup_rows_per_dst=[5424, 5614, 5522, 5418, 5346, 5520, 5480, 5460] combine_rows_per_src=[11034, 11060, 10892, 10688, 10856, 10992, 10980, 10786] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87282 payload_rows_remote=76398 dup_rows=43790 dup_rows_remote=38246 dedup_ratio=0.334091 router_tokens_per_expert=[468, 532, 548, 527, 476, 523, 473, 468, 541, 528, 541, 508, 507, 508, 503, 541, 476, 479, 516, 515, 496, 507, 527, 508, 560, 526, 523, 516, 466, 554, 522, 498, 519, 508, 512, 514, 519, 529, 522, 505, 527, 527, 490, 506, 527, 523, 468, 470, 512, 515, 542, 497, 472, 500, 527, 538, 512, 495, 523, 493, 500, 476, 513, 507, 543, 493, 504, 513, 461, 535, 514, 505, 481, 522, 494, 554, 530, 487, 495, 494, 495, 492, 505, 548, 541, 486, 485, 492, 512, 546, 512, 526, 520, 486, 520, 510, 524, 514, 521, 498, 524, 527, 461, 548, 509, 533, 525, 519, 529, 495, 495, 492, 477, 527, 536, 572, 494, 471, 533, 508, 575, 497, 522, 531, 536, 569, 487, 517] payload_rows_per_dst=[10934, 10906, 10856, 10896, 10880, 10886, 10884, 11040] dup_rows_per_dst=[5450, 5472, 5476, 5348, 5370, 5466, 5544, 5664] combine_rows_per_src=[10934, 10906, 10856, 10896, 10880, 10886, 10884, 11040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628097024 dispatch_meta_bytes=458872 combine_payload_bytes=628097024 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628555896 ingress_bytes=779091968 link_bytes=779091968 token_bytes=1256652920 weight_movement_bytes=150994944 total_nvlink_bytes=1407647864 weight_to_token_ratio=0.120156 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.879982809e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87296 payload_rows_remote=76370 dup_rows=43776 dup_rows_remote=38292 dedup_ratio=0.333984 router_tokens_per_expert=[540, 495, 526, 515, 518, 539, 467, 558, 530, 515, 506, 491, 533, 519, 500, 513, 481, 560, 486, 541, 514, 530, 511, 551, 460, 508, 513, 476, 493, 474, 490, 540, 510, 500, 524, 503, 521, 515, 537, 536, 520, 523, 488, 501, 545, 528, 493, 505, 492, 524, 513, 460, 467, 503, 548, 506, 540, 521, 535, 521, 492, 525, 502, 520, 512, 549, 583, 514, 516, 476, 512, 531, 508, 489, 509, 513, 512, 490, 529, 516, 515, 516, 525, 503, 511, 514, 495, 520, 511, 499, 505, 508, 562, 485, 513, 523, 563, 505, 502, 509, 528, 537, 530, 487, 525, 478, 502, 506, 531, 500, 466, 509, 472, 510, 464, 498, 500, 496, 486, 501, 557, 530, 516, 485, 537, 557, 473, 501] payload_rows_per_dst=[10868, 10900, 10986, 10902, 11008, 10926, 10876, 10830] dup_rows_per_dst=[5662, 5356, 5512, 5436, 5510, 5484, 5480, 5336] combine_rows_per_src=[10868, 10900, 10986, 10902, 11008, 10926, 10876, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87734 payload_rows_remote=76612 dup_rows=43338 dup_rows_remote=37526 dedup_ratio=0.330643 router_tokens_per_expert=[491, 500, 521, 527, 497, 514, 522, 504, 531, 526, 520, 482, 495, 497, 529, 514, 476, 539, 514, 498, 497, 503, 518, 505, 542, 492, 469, 511, 504, 499, 519, 525, 515, 484, 479, 514, 519, 503, 573, 549, 497, 507, 471, 499, 526, 506, 482, 525, 492, 514, 529, 494, 500, 525, 481, 475, 520, 498, 547, 519, 561, 544, 492, 510, 508, 523, 515, 505, 511, 536, 520, 535, 520, 507, 516, 519, 542, 508, 514, 493, 506, 548, 529, 505, 521, 519, 548, 505, 502, 537, 530, 491, 503, 532, 482, 476, 501, 522, 471, 541, 486, 510, 518, 539, 501, 526, 522, 548, 502, 528, 507, 548, 516, 493, 498, 509, 532, 510, 530, 501, 495, 543, 522, 522, 470, 512, 502, 474] payload_rows_per_dst=[10894, 10904, 10868, 11010, 11090, 11034, 10812, 11122] dup_rows_per_dst=[4946, 5318, 5504, 5674, 5454, 5442, 5188, 5812] combine_rows_per_src=[10894, 10904, 10868, 11010, 11090, 11034, 10812, 11122] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87474 payload_rows_remote=76598 dup_rows=43598 dup_rows_remote=38072 dedup_ratio=0.332626 router_tokens_per_expert=[523, 473, 514, 488, 510, 506, 528, 524, 531, 520, 521, 507, 510, 520, 527, 558, 461, 541, 492, 546, 503, 544, 501, 488, 528, 463, 535, 525, 519, 502, 501, 535, 500, 462, 540, 479, 533, 495, 480, 507, 503, 516, 543, 511, 531, 469, 494, 541, 507, 535, 546, 523, 515, 505, 497, 536, 498, 510, 475, 501, 515, 464, 551, 523, 533, 492, 514, 487, 513, 489, 510, 522, 538, 492, 526, 532, 476, 496, 508, 524, 517, 530, 507, 535, 464, 524, 547, 523, 515, 470, 524, 503, 512, 494, 534, 490, 542, 517, 492, 485, 545, 492, 557, 510, 485, 513, 534, 529, 502, 551, 494, 506, 483, 521, 512, 523, 489, 534, 517, 505, 503, 498, 485, 546, 486, 491, 552, 547] payload_rows_per_dst=[10958, 10916, 10816, 10876, 10946, 10948, 11066, 10948] dup_rows_per_dst=[5562, 5452, 5392, 5526, 5358, 5430, 5442, 5436] combine_rows_per_src=[10958, 10916, 10816, 10876, 10946, 10948, 11066, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=458656 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627212192 ingress_bytes=702251008 link_bytes=702251008 token_bytes=1253965728 weight_movement_bytes=75497472 total_nvlink_bytes=1329463200 weight_to_token_ratio=0.060207 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905531274e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458632 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626294664 ingress_bytes=701333504 link_bytes=701333504 token_bytes=1252130696 weight_movement_bytes=75497472 total_nvlink_bytes=1327628168 weight_to_token_ratio=0.060295 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.893896032e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625852416 dispatch_meta_bytes=458576 combine_payload_bytes=625852416 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626310992 ingress_bytes=776847360 link_bytes=776847360 token_bytes=1252163408 weight_movement_bytes=150994944 total_nvlink_bytes=1403158352 weight_to_token_ratio=0.120587 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.851518020e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625623040 dispatch_meta_bytes=458648 combine_payload_bytes=625623040 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626081688 ingress_bytes=852115456 link_bytes=852115456 token_bytes=1251704728 weight_movement_bytes=226492416 total_nvlink_bytes=1478197144 weight_to_token_ratio=0.180947 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080602343e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627605504 dispatch_meta_bytes=456552 combine_payload_bytes=627605504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628062056 ingress_bytes=703102976 link_bytes=703102976 token_bytes=1255667560 weight_movement_bytes=75497472 total_nvlink_bytes=1331165032 weight_to_token_ratio=0.060125 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916335428e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627490816 dispatch_meta_bytes=458680 combine_payload_bytes=627490816 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627949496 ingress_bytes=702988288 link_bytes=702988288 token_bytes=1255440312 weight_movement_bytes=75497472 total_nvlink_bytes=1330937784 weight_to_token_ratio=0.060136 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914881023e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87688 payload_rows_remote=76824 dup_rows=43384 dup_rows_remote=38064 dedup_ratio=0.330994 router_tokens_per_expert=[496, 540, 497, 494, 484, 512, 530, 492, 551, 529, 509, 474, 508, 519, 536, 517, 510, 548, 518, 499, 569, 494, 527, 519, 532, 474, 517, 515, 506, 533, 503, 530, 527, 525, 510, 488, 525, 483, 522, 474, 532, 484, 514, 504, 489, 540, 493, 577, 493, 488, 530, 501, 486, 486, 510, 522, 530, 528, 526, 547, 534, 515, 517, 482, 514, 517, 517, 529, 507, 497, 498, 498, 492, 548, 512, 469, 459, 518, 488, 529, 501, 489, 514, 495, 497, 506, 511, 548, 488, 501, 510, 525, 515, 516, 519, 491, 497, 502, 540, 535, 516, 529, 513, 529, 515, 524, 502, 485, 537, 499, 513, 552, 491, 515, 510, 532, 482, 483, 545, 546, 514, 517, 499, 495, 486, 525, 520, 506] payload_rows_per_dst=[10972, 11042, 10888, 10956, 10864, 10914, 11122, 10930] dup_rows_per_dst=[5404, 5546, 5486, 5434, 5320, 5338, 5454, 5402] combine_rows_per_src=[10972, 11042, 10888, 10956, 10864, 10914, 11122, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87420 payload_rows_remote=76482 dup_rows=43652 dup_rows_remote=38110 dedup_ratio=0.333038 router_tokens_per_expert=[531, 545, 487, 508, 491, 544, 503, 531, 492, 539, 492, 516, 525, 529, 501, 506, 487, 508, 520, 523, 508, 491, 508, 529, 501, 495, 501, 487, 515, 481, 533, 485, 488, 474, 521, 507, 500, 536, 500, 536, 507, 492, 491, 525, 505, 502, 513, 566, 536, 540, 501, 514, 508, 547, 551, 516, 509, 495, 528, 536, 514, 532, 537, 468, 537, 510, 507, 502, 488, 507, 512, 479, 539, 486, 527, 514, 538, 579, 544, 501, 524, 471, 520, 480, 513, 498, 531, 531, 562, 532, 515, 455, 544, 518, 519, 548, 498, 488, 526, 474, 502, 550, 505, 481, 510, 488, 558, 499, 508, 512, 523, 538, 509, 494, 480, 487, 502, 549, 494, 505, 472, 515, 490, 505, 524, 511, 493, 508] payload_rows_per_dst=[10938, 10832, 10932, 11016, 11012, 10932, 10942, 10816] dup_rows_per_dst=[5542, 5346, 5394, 5648, 5646, 5438, 5378, 5260] combine_rows_per_src=[10938, 10832, 10932, 11016, 11012, 10932, 10942, 10816] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87368 payload_rows_remote=76496 dup_rows=43704 dup_rows_remote=38296 dedup_ratio=0.333435 router_tokens_per_expert=[511, 505, 515, 502, 508, 504, 529, 570, 490, 466, 535, 538, 493, 585, 516, 482, 509, 491, 491, 493, 529, 523, 518, 496, 499, 544, 504, 512, 528, 479, 526, 498, 514, 516, 497, 477, 488, 511, 489, 542, 512, 548, 506, 504, 537, 506, 531, 510, 508, 501, 535, 528, 503, 496, 501, 539, 476, 494, 518, 526, 549, 504, 518, 506, 527, 478, 510, 505, 505, 493, 517, 540, 521, 515, 548, 482, 549, 530, 495, 507, 483, 518, 528, 515, 491, 500, 488, 500, 479, 517, 513, 558, 523, 489, 507, 505, 528, 501, 503, 510, 505, 535, 489, 491, 506, 499, 523, 493, 523, 457, 539, 524, 460, 530, 535, 537, 532, 511, 526, 532, 532, 513, 491, 517, 518, 540, 511, 510] payload_rows_per_dst=[10972, 10872, 10916, 10990, 10964, 10822, 10828, 11004] dup_rows_per_dst=[5526, 5408, 5460, 5414, 5480, 5406, 5424, 5586] combine_rows_per_src=[10972, 10872, 10916, 10990, 10964, 10822, 10828, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=629342208 dispatch_meta_bytes=459552 combine_payload_bytes=629342208 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629801760 ingress_bytes=704839680 link_bytes=704839680 token_bytes=1259143968 weight_movement_bytes=75497472 total_nvlink_bytes=1334641440 weight_to_token_ratio=0.059959 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.938359279e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76638 dup_rows=43570 dup_rows_remote=38156 dedup_ratio=0.332413 router_tokens_per_expert=[558, 527, 512, 475, 516, 560, 489, 484, 524, 501, 497, 531, 506, 527, 489, 528, 540, 538, 476, 512, 496, 480, 506, 520, 525, 498, 532, 509, 538, 561, 559, 538, 487, 474, 512, 463, 523, 526, 504, 515, 444, 552, 574, 534, 470, 534, 479, 548, 536, 518, 557, 504, 534, 584, 465, 520, 519, 510, 506, 531, 535, 486, 489, 530, 448, 523, 474, 536, 519, 498, 511, 496, 496, 507, 514, 531, 529, 483, 491, 504, 559, 514, 503, 492, 521, 529, 491, 527, 491, 491, 464, 540, 515, 501, 520, 466, 509, 492, 536, 531, 540, 518, 515, 456, 503, 533, 501, 472, 505, 551, 549, 526, 490, 487, 531, 516, 499, 477, 535, 555, 458, 502, 494, 507, 502, 501, 562, 484] payload_rows_per_dst=[11048, 11060, 10864, 11048, 10844, 10862, 10938, 10838] dup_rows_per_dst=[5400, 5596, 5414, 5600, 5276, 5386, 5536, 5362] combine_rows_per_src=[11048, 11060, 10864, 11048, 10844, 10862, 10938, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87100 payload_rows_remote=76296 dup_rows=43972 dup_rows_remote=38640 dedup_ratio=0.335480 router_tokens_per_expert=[521, 520, 517, 522, 514, 479, 519, 474, 523, 521, 583, 498, 506, 502, 500, 548, 539, 494, 507, 523, 555, 513, 487, 522, 550, 488, 520, 496, 579, 535, 515, 532, 539, 498, 521, 476, 499, 494, 544, 499, 517, 514, 530, 498, 496, 489, 505, 490, 501, 486, 495, 520, 485, 541, 516, 497, 495, 464, 504, 509, 532, 513, 525, 502, 502, 501, 520, 504, 511, 555, 533, 486, 544, 505, 539, 539, 553, 526, 512, 494, 505, 492, 538, 462, 472, 519, 529, 506, 507, 522, 509, 497, 538, 486, 494, 545, 492, 522, 511, 494, 479, 530, 456, 525, 533, 504, 504, 506, 506, 529, 483, 494, 517, 511, 542, 545, 502, 504, 491, 504, 543, 476, 511, 470, 495, 541, 539, 536] payload_rows_per_dst=[10840, 11050, 10802, 10788, 10996, 10790, 10804, 11030] dup_rows_per_dst=[5654, 5660, 5416, 5382, 5652, 5452, 5332, 5424] combine_rows_per_src=[10840, 11050, 10802, 10788, 10996, 10790, 10804, 11030] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87584 payload_rows_remote=76804 dup_rows=43488 dup_rows_remote=38124 dedup_ratio=0.331787 router_tokens_per_expert=[549, 495, 540, 499, 561, 534, 529, 500, 506, 547, 496, 497, 517, 527, 532, 535, 500, 508, 473, 476, 503, 503, 549, 513, 523, 503, 497, 520, 514, 537, 541, 467, 534, 508, 511, 504, 541, 492, 483, 513, 531, 497, 486, 487, 494, 542, 516, 503, 527, 481, 507, 509, 519, 531, 527, 517, 473, 506, 535, 522, 498, 511, 505, 479, 501, 507, 506, 470, 516, 533, 505, 483, 513, 543, 520, 500, 541, 560, 467, 506, 502, 483, 520, 524, 488, 500, 490, 492, 525, 526, 507, 487, 496, 483, 532, 517, 547, 522, 520, 504, 519, 546, 536, 500, 478, 514, 488, 511, 513, 519, 507, 548, 515, 501, 517, 506, 514, 521, 501, 549, 515, 536, 541, 520, 499, 510, 515, 481] payload_rows_per_dst=[11142, 11004, 10902, 10946, 10878, 10780, 11030, 10902] dup_rows_per_dst=[5586, 5250, 5382, 5348, 5464, 5364, 5514, 5580] combine_rows_per_src=[11142, 11004, 10902, 10946, 10878, 10780, 11030, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626540544 dispatch_meta_bytes=458368 combine_payload_bytes=626540544 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=626998912 ingress_bytes=928530432 link_bytes=928530432 token_bytes=1253539456 weight_movement_bytes=301989888 total_nvlink_bytes=1555529344 weight_to_token_ratio=0.240910 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177507288e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87734 payload_rows_remote=76662 dup_rows=43338 dup_rows_remote=37928 dedup_ratio=0.330643 router_tokens_per_expert=[521, 495, 481, 489, 488, 496, 523, 502, 519, 493, 546, 512, 513, 536, 533, 528, 550, 547, 532, 540, 457, 505, 498, 484, 517, 498, 522, 572, 516, 490, 455, 543, 483, 546, 519, 510, 506, 496, 494, 520, 481, 494, 517, 534, 523, 542, 565, 512, 491, 543, 495, 469, 478, 522, 502, 484, 515, 502, 537, 502, 543, 521, 549, 502, 528, 489, 511, 518, 484, 530, 505, 515, 523, 548, 496, 486, 523, 530, 531, 502, 450, 517, 536, 512, 506, 506, 510, 490, 523, 526, 512, 512, 523, 531, 496, 496, 503, 533, 522, 484, 490, 535, 558, 509, 533, 532, 489, 482, 508, 512, 497, 460, 524, 510, 548, 527, 515, 548, 536, 501, 499, 492, 546, 503, 465, 487, 500, 525] payload_rows_per_dst=[10726, 10878, 10994, 10922, 11076, 11098, 10968, 11072] dup_rows_per_dst=[5070, 5262, 5490, 5392, 5650, 5738, 5326, 5410] combine_rows_per_src=[10726, 10878, 10994, 10922, 11076, 11098, 10968, 11072] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87334 payload_rows_remote=76392 dup_rows=43738 dup_rows_remote=38330 dedup_ratio=0.333694 router_tokens_per_expert=[557, 510, 520, 473, 519, 563, 526, 501, 497, 511, 508, 502, 509, 545, 504, 490, 519, 499, 518, 494, 530, 483, 523, 494, 506, 521, 537, 544, 559, 498, 529, 491, 479, 515, 521, 507, 539, 490, 526, 504, 512, 473, 532, 489, 517, 501, 517, 544, 509, 475, 500, 532, 479, 539, 519, 496, 527, 504, 557, 503, 513, 504, 521, 497, 525, 517, 477, 530, 461, 503, 549, 526, 523, 503, 537, 499, 489, 547, 484, 541, 521, 509, 478, 502, 487, 463, 536, 522, 525, 497, 517, 493, 504, 536, 506, 508, 496, 518, 535, 497, 524, 550, 522, 533, 532, 505, 503, 472, 519, 500, 513, 496, 506, 529, 498, 530, 523, 541, 513, 531, 508, 495, 493, 495, 490, 511, 512, 510] payload_rows_per_dst=[10918, 10924, 10808, 10942, 10980, 10882, 10952, 10928] dup_rows_per_dst=[5552, 5566, 5524, 5408, 5442, 5326, 5478, 5442] combine_rows_per_src=[10918, 10924, 10808, 10942, 10980, 10882, 10952, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626655232 dispatch_meta_bytes=459168 combine_payload_bytes=626655232 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627114400 ingress_bytes=853147648 link_bytes=853147648 token_bytes=1253769632 weight_movement_bytes=226492416 total_nvlink_bytes=1480262048 weight_to_token_ratio=0.180649 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081911307e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627818496 dispatch_meta_bytes=459176 combine_payload_bytes=627818496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628277672 ingress_bytes=703315968 link_bytes=703315968 token_bytes=1256096168 weight_movement_bytes=75497472 total_nvlink_bytes=1331593640 weight_to_token_ratio=0.060105 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919036466e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625016832 dispatch_meta_bytes=459744 combine_payload_bytes=625016832 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625476576 ingress_bytes=700514304 link_bytes=700514304 token_bytes=1250493408 weight_movement_bytes=75497472 total_nvlink_bytes=1325990880 weight_to_token_ratio=0.060374 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.883507423e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=629178368 dispatch_meta_bytes=459712 combine_payload_bytes=629178368 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629638080 ingress_bytes=704675840 link_bytes=704675840 token_bytes=1258816448 weight_movement_bytes=75497472 total_nvlink_bytes=1334313920 weight_to_token_ratio=0.059975 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.936281557e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628015104 dispatch_meta_bytes=458360 combine_payload_bytes=628015104 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628473464 ingress_bytes=779010048 link_bytes=779010048 token_bytes=1256488568 weight_movement_bytes=150994944 total_nvlink_bytes=1407483512 weight_to_token_ratio=0.120172 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.878943948e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625803264 dispatch_meta_bytes=458888 combine_payload_bytes=625803264 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626262152 ingress_bytes=776798208 link_bytes=776798208 token_bytes=1252065416 weight_movement_bytes=150994944 total_nvlink_bytes=1403060360 weight_to_token_ratio=0.120597 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.850894704e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87764 payload_rows_remote=76836 dup_rows=43308 dup_rows_remote=37870 dedup_ratio=0.330414 router_tokens_per_expert=[495, 507, 478, 531, 503, 513, 519, 532, 484, 509, 503, 529, 531, 523, 515, 494, 514, 529, 519, 519, 501, 496, 508, 492, 520, 465, 505, 523, 511, 535, 478, 483, 475, 526, 556, 529, 491, 508, 515, 550, 506, 502, 505, 513, 462, 509, 504, 543, 501, 503, 515, 532, 525, 494, 561, 509, 531, 503, 487, 545, 496, 518, 515, 511, 502, 503, 519, 484, 525, 512, 539, 531, 500, 525, 513, 472, 531, 514, 521, 492, 507, 525, 525, 548, 530, 541, 519, 521, 494, 519, 533, 508, 517, 481, 522, 511, 485, 528, 526, 521, 563, 521, 497, 480, 524, 508, 494, 500, 495, 489, 532, 504, 525, 512, 534, 494, 535, 538, 517, 491, 513, 517, 485, 478, 516, 509, 490, 527] payload_rows_per_dst=[10954, 10834, 10958, 11008, 10928, 11096, 10964, 11022] dup_rows_per_dst=[5378, 5362, 5430, 5484, 5438, 5506, 5370, 5340] combine_rows_per_src=[10954, 10834, 10958, 11008, 10928, 11096, 10964, 11022] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=629440512 dispatch_meta_bytes=458824 combine_payload_bytes=629440512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629899336 ingress_bytes=704937984 link_bytes=704937984 token_bytes=1259339848 weight_movement_bytes=75497472 total_nvlink_bytes=1334837320 weight_to_token_ratio=0.059950 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.939605912e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87446 payload_rows_remote=76528 dup_rows=43626 dup_rows_remote=38238 dedup_ratio=0.332840 router_tokens_per_expert=[493, 497, 521, 498, 524, 546, 466, 494, 477, 535, 528, 484, 544, 497, 508, 541, 539, 525, 510, 480, 533, 501, 539, 540, 496, 498, 510, 509, 520, 482, 509, 482, 552, 505, 502, 484, 545, 535, 507, 513, 523, 521, 514, 518, 533, 521, 524, 499, 544, 513, 492, 512, 540, 489, 503, 509, 546, 524, 495, 531, 539, 504, 529, 490, 471, 539, 479, 489, 500, 508, 491, 470, 499, 535, 531, 497, 474, 517, 522, 480, 538, 525, 481, 516, 535, 570, 502, 531, 513, 482, 524, 508, 526, 499, 514, 518, 516, 526, 459, 492, 521, 509, 528, 498, 520, 534, 505, 532, 538, 544, 543, 491, 515, 497, 484, 504, 510, 530, 492, 508, 491, 523, 496, 520, 536, 522, 506, 480] payload_rows_per_dst=[10918, 10944, 11068, 10902, 10814, 10974, 10990, 10836] dup_rows_per_dst=[5388, 5402, 5644, 5498, 5190, 5590, 5618, 5296] combine_rows_per_src=[10918, 10944, 11068, 10902, 10814, 10974, 10990, 10836] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76770 dup_rows=43506 dup_rows_remote=38256 dedup_ratio=0.331924 router_tokens_per_expert=[540, 498, 490, 486, 489, 553, 521, 503, 511, 496, 523, 527, 512, 507, 496, 515, 490, 471, 501, 502, 521, 492, 524, 523, 529, 495, 520, 514, 483, 485, 472, 501, 483, 504, 533, 529, 536, 539, 530, 540, 515, 550, 492, 509, 523, 532, 514, 529, 497, 514, 504, 545, 513, 525, 497, 494, 467, 480, 544, 557, 496, 540, 514, 524, 471, 509, 502, 533, 530, 524, 499, 490, 537, 517, 502, 492, 518, 530, 475, 505, 507, 505, 541, 490, 528, 508, 534, 501, 530, 489, 495, 516, 530, 497, 547, 525, 491, 515, 496, 512, 491, 555, 565, 499, 526, 458, 562, 537, 509, 496, 536, 513, 499, 527, 516, 459, 495, 541, 511, 496, 529, 564, 519, 556, 493, 492, 471, 471] payload_rows_per_dst=[10970, 10796, 11106, 10860, 10890, 10990, 11012, 10942] dup_rows_per_dst=[5364, 5250, 5610, 5562, 5378, 5496, 5510, 5336] combine_rows_per_src=[10970, 10796, 11106, 10860, 10890, 10990, 11012, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87286 payload_rows_remote=76180 dup_rows=43786 dup_rows_remote=38240 dedup_ratio=0.334061 router_tokens_per_expert=[504, 512, 475, 475, 462, 442, 527, 547, 552, 525, 538, 550, 511, 521, 521, 470, 496, 480, 509, 523, 498, 515, 517, 544, 506, 522, 522, 475, 486, 526, 497, 523, 537, 507, 516, 507, 530, 500, 553, 498, 474, 546, 551, 535, 507, 524, 502, 539, 524, 544, 527, 503, 491, 575, 503, 503, 545, 567, 493, 538, 498, 507, 498, 492, 512, 487, 503, 511, 516, 481, 518, 489, 538, 492, 514, 552, 519, 503, 504, 526, 502, 507, 525, 531, 526, 519, 504, 514, 537, 471, 503, 477, 488, 513, 509, 509, 488, 482, 531, 499, 520, 505, 497, 506, 497, 544, 459, 513, 538, 557, 524, 517, 507, 475, 545, 483, 503, 481, 525, 513, 535, 528, 491, 542, 484, 523, 524, 495] payload_rows_per_dst=[10852, 10838, 11106, 11026, 10962, 10728, 10862, 10912] dup_rows_per_dst=[5412, 5440, 5546, 5590, 5368, 5542, 5492, 5396] combine_rows_per_src=[10852, 10838, 11106, 11026, 10962, 10728, 10862, 10912] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87502 payload_rows_remote=76464 dup_rows=43570 dup_rows_remote=38116 dedup_ratio=0.332413 router_tokens_per_expert=[506, 539, 488, 522, 517, 527, 501, 520, 546, 506, 510, 559, 509, 471, 564, 508, 519, 482, 531, 500, 515, 487, 517, 506, 511, 516, 510, 567, 512, 507, 499, 491, 531, 537, 525, 480, 471, 485, 494, 509, 498, 494, 515, 516, 477, 515, 502, 518, 489, 528, 492, 526, 528, 499, 500, 515, 505, 478, 459, 503, 502, 536, 518, 551, 493, 496, 548, 503, 541, 528, 493, 506, 467, 532, 515, 502, 502, 535, 506, 507, 500, 501, 490, 518, 535, 497, 519, 521, 510, 492, 486, 533, 537, 510, 501, 535, 539, 519, 510, 512, 536, 536, 524, 517, 487, 511, 517, 496, 486, 542, 536, 478, 544, 543, 553, 513, 483, 482, 497, 551, 544, 540, 495, 512, 514, 492, 507, 502] payload_rows_per_dst=[11012, 10924, 10836, 10758, 10942, 11020, 11038, 10972] dup_rows_per_dst=[5574, 5416, 5298, 5500, 5406, 5350, 5454, 5572] combine_rows_per_src=[11012, 10924, 10836, 10758, 10942, 11020, 11038, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87292 payload_rows_remote=76550 dup_rows=43780 dup_rows_remote=38374 dedup_ratio=0.334015 router_tokens_per_expert=[471, 485, 525, 533, 495, 489, 487, 518, 553, 537, 543, 501, 483, 487, 533, 477, 506, 560, 504, 491, 540, 512, 565, 523, 560, 497, 515, 521, 511, 551, 500, 508, 544, 494, 497, 531, 496, 489, 467, 543, 538, 488, 564, 565, 507, 507, 480, 531, 529, 515, 505, 510, 489, 536, 514, 531, 470, 524, 514, 492, 528, 505, 502, 515, 528, 473, 494, 534, 549, 493, 502, 482, 522, 546, 536, 487, 501, 497, 526, 494, 509, 532, 524, 459, 511, 484, 511, 506, 481, 479, 507, 527, 493, 512, 529, 510, 534, 485, 506, 470, 514, 490, 508, 491, 532, 489, 503, 498, 516, 572, 528, 494, 512, 490, 502, 540, 483, 534, 525, 546, 504, 528, 530, 520, 538, 515, 514, 486] payload_rows_per_dst=[10906, 11060, 10906, 10882, 10886, 10742, 10862, 11048] dup_rows_per_dst=[5328, 5668, 5576, 5476, 5442, 5406, 5398, 5486] combine_rows_per_src=[10906, 11060, 10906, 10882, 10886, 10742, 10862, 11048] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87556 payload_rows_remote=76604 dup_rows=43516 dup_rows_remote=38110 dedup_ratio=0.332001 router_tokens_per_expert=[461, 473, 500, 490, 534, 487, 494, 518, 516, 553, 496, 515, 498, 517, 465, 548, 509, 488, 521, 508, 529, 523, 491, 497, 515, 472, 506, 580, 541, 512, 495, 516, 541, 522, 505, 535, 513, 500, 501, 532, 521, 538, 514, 513, 519, 487, 524, 501, 540, 533, 516, 501, 519, 499, 506, 533, 533, 501, 462, 517, 543, 490, 515, 502, 507, 521, 491, 500, 517, 512, 507, 553, 488, 532, 530, 473, 517, 534, 474, 504, 500, 505, 483, 519, 485, 478, 510, 465, 548, 505, 552, 523, 536, 546, 555, 514, 515, 509, 506, 494, 536, 480, 492, 494, 498, 517, 541, 520, 580, 495, 525, 527, 504, 544, 520, 529, 528, 509, 457, 506, 511, 502, 529, 513, 510, 506, 508, 503] payload_rows_per_dst=[10842, 10920, 10916, 11040, 10856, 11072, 10958, 10952] dup_rows_per_dst=[5288, 5486, 5616, 5380, 5464, 5376, 5500, 5406] combine_rows_per_src=[10842, 10920, 10916, 11040, 10856, 11072, 10958, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626917376 dispatch_meta_bytes=459064 combine_payload_bytes=626917376 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627376440 ingress_bytes=853409792 link_bytes=853409792 token_bytes=1254293816 weight_movement_bytes=226492416 total_nvlink_bytes=1480786232 weight_to_token_ratio=0.180574 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082243743e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=628899840 dispatch_meta_bytes=460104 combine_payload_bytes=628899840 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629359944 ingress_bytes=704397312 link_bytes=704397312 token_bytes=1258259784 weight_movement_bytes=75497472 total_nvlink_bytes=1333757256 weight_to_token_ratio=0.060001 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.932749430e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87400 payload_rows_remote=76510 dup_rows=43672 dup_rows_remote=38164 dedup_ratio=0.333191 router_tokens_per_expert=[520, 516, 502, 520, 486, 477, 482, 514, 507, 485, 531, 497, 556, 536, 529, 508, 508, 486, 484, 528, 497, 502, 520, 536, 503, 526, 530, 539, 506, 509, 498, 499, 475, 537, 523, 507, 484, 495, 500, 502, 519, 494, 540, 522, 495, 519, 509, 533, 509, 530, 519, 479, 512, 490, 473, 515, 537, 523, 482, 522, 499, 519, 559, 531, 503, 480, 527, 511, 514, 506, 508, 487, 526, 524, 505, 493, 542, 535, 494, 508, 543, 537, 530, 525, 501, 535, 502, 514, 482, 489, 514, 507, 483, 526, 506, 504, 523, 504, 524, 461, 552, 497, 503, 495, 516, 543, 542, 486, 470, 543, 508, 577, 526, 522, 520, 516, 520, 539, 499, 485, 524, 482, 506, 490, 527, 547, 557, 481] payload_rows_per_dst=[10790, 10928, 10944, 10890, 10928, 10970, 10952, 10998] dup_rows_per_dst=[5542, 5414, 5364, 5508, 5398, 5426, 5536, 5484] combine_rows_per_src=[10790, 10928, 10944, 10890, 10928, 10970, 10952, 10998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624066560 dispatch_meta_bytes=457680 combine_payload_bytes=624066560 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=624524240 ingress_bytes=775061504 link_bytes=775061504 token_bytes=1248590800 weight_movement_bytes=150994944 total_nvlink_bytes=1399585744 weight_to_token_ratio=0.120932 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.828870852e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626393088 dispatch_meta_bytes=458320 combine_payload_bytes=626393088 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626851408 ingress_bytes=852885504 link_bytes=852885504 token_bytes=1253244496 weight_movement_bytes=226492416 total_nvlink_bytes=1479736912 weight_to_token_ratio=0.180725 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081578872e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627097600 dispatch_meta_bytes=459696 combine_payload_bytes=627097600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627557296 ingress_bytes=702595072 link_bytes=702595072 token_bytes=1254654896 weight_movement_bytes=75497472 total_nvlink_bytes=1330152368 weight_to_token_ratio=0.060174 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909894490e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=458856 combine_payload_bytes=627539968 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627998824 ingress_bytes=778534912 link_bytes=778534912 token_bytes=1255538792 weight_movement_bytes=150994944 total_nvlink_bytes=1406533736 weight_to_token_ratio=0.120263 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872918555e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76342 dup_rows=43666 dup_rows_remote=38138 dedup_ratio=0.333145 router_tokens_per_expert=[518, 521, 515, 497, 520, 523, 510, 526, 493, 560, 516, 522, 490, 534, 501, 485, 540, 518, 526, 525, 548, 512, 529, 517, 492, 492, 532, 505, 527, 497, 494, 521, 506, 535, 463, 534, 509, 530, 507, 525, 490, 526, 465, 512, 504, 543, 520, 510, 467, 481, 467, 533, 494, 507, 497, 497, 476, 510, 471, 513, 565, 528, 552, 520, 514, 503, 536, 521, 521, 530, 538, 513, 543, 520, 495, 517, 524, 492, 517, 512, 532, 501, 493, 506, 510, 492, 506, 504, 508, 520, 526, 487, 526, 501, 520, 524, 510, 529, 518, 498, 530, 528, 473, 491, 511, 583, 494, 546, 505, 493, 526, 483, 518, 490, 536, 489, 520, 487, 514, 507, 510, 502, 492, 510, 503, 484, 524, 517] payload_rows_per_dst=[11056, 10922, 10930, 10844, 11064, 10848, 10886, 10856] dup_rows_per_dst=[5406, 5628, 5428, 5312, 5528, 5464, 5550, 5350] combine_rows_per_src=[11056, 10922, 10930, 10844, 11064, 10848, 10886, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626769920 dispatch_meta_bytes=458696 combine_payload_bytes=626769920 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627228616 ingress_bytes=777764864 link_bytes=777764864 token_bytes=1253998536 weight_movement_bytes=150994944 total_nvlink_bytes=1404993480 weight_to_token_ratio=0.120411 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.863153262e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=625393664 dispatch_meta_bytes=457920 combine_payload_bytes=625393664 prefetch_bytes=0 reduce_bytes=0 egress_bytes=625851584 ingress_bytes=625393664 link_bytes=625851584 token_bytes=1251245248 weight_movement_bytes=0 total_nvlink_bytes=1251245248 weight_to_token_ratio=0.000000 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=7.936679038e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87324 payload_rows_remote=76236 dup_rows=43748 dup_rows_remote=38106 dedup_ratio=0.333771 router_tokens_per_expert=[536, 542, 502, 509, 513, 513, 500, 556, 492, 480, 535, 504, 491, 483, 495, 529, 523, 503, 502, 512, 518, 537, 485, 485, 494, 515, 489, 532, 489, 502, 493, 534, 480, 508, 476, 505, 503, 477, 515, 516, 514, 515, 565, 500, 520, 541, 493, 481, 504, 490, 504, 505, 534, 539, 499, 529, 510, 506, 516, 524, 527, 515, 528, 512, 491, 522, 492, 539, 507, 517, 558, 519, 495, 494, 498, 514, 528, 529, 514, 563, 525, 518, 497, 448, 482, 521, 521, 552, 542, 515, 522, 503, 518, 485, 579, 504, 523, 486, 520, 494, 520, 520, 494, 464, 536, 518, 498, 540, 490, 552, 502, 529, 514, 526, 503, 526, 501, 516, 526, 513, 483, 505, 528, 512, 501, 522, 521, 497] payload_rows_per_dst=[11088, 10868, 10774, 10984, 11010, 11008, 10768, 10824] dup_rows_per_dst=[5642, 5514, 5116, 5460, 5550, 5634, 5294, 5538] combine_rows_per_src=[11088, 10868, 10774, 10984, 11010, 11008, 10768, 10824] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87716 payload_rows_remote=76886 dup_rows=43356 dup_rows_remote=38018 dedup_ratio=0.330780 router_tokens_per_expert=[475, 501, 497, 536, 500, 461, 497, 508, 544, 524, 545, 544, 521, 518, 483, 556, 542, 518, 515, 547, 493, 502, 495, 499, 527, 521, 478, 508, 515, 475, 477, 464, 550, 531, 466, 492, 508, 498, 500, 557, 532, 523, 546, 506, 505, 501, 534, 522, 503, 530, 477, 485, 514, 478, 529, 518, 519, 502, 529, 521, 494, 544, 520, 558, 500, 545, 498, 519, 514, 508, 481, 505, 511, 514, 544, 484, 531, 510, 505, 528, 489, 519, 503, 501, 524, 502, 515, 530, 484, 544, 505, 566, 491, 520, 506, 500, 524, 525, 518, 475, 485, 547, 526, 498, 523, 511, 499, 511, 516, 480, 527, 484, 561, 487, 510, 514, 517, 529, 499, 516, 508, 496, 511, 530, 521, 466, 515, 533] payload_rows_per_dst=[10912, 10830, 11036, 11050, 10908, 11014, 10934, 11032] dup_rows_per_dst=[5508, 5338, 5506, 5376, 5486, 5384, 5364, 5394] combine_rows_per_src=[10912, 10830, 11036, 11050, 10908, 11014, 10934, 11032] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87578 payload_rows_remote=76702 dup_rows=43494 dup_rows_remote=37996 dedup_ratio=0.331833 router_tokens_per_expert=[523, 493, 526, 504, 495, 518, 510, 516, 471, 506, 547, 495, 539, 533, 498, 512, 529, 472, 520, 526, 492, 541, 474, 470, 526, 506, 489, 539, 489, 500, 547, 490, 503, 505, 493, 525, 513, 514, 534, 502, 490, 491, 509, 558, 529, 544, 467, 510, 527, 516, 509, 535, 517, 498, 487, 562, 526, 533, 529, 485, 524, 490, 529, 489, 498, 483, 489, 497, 492, 530, 520, 516, 483, 519, 527, 509, 491, 511, 529, 493, 474, 511, 521, 486, 536, 518, 566, 511, 533, 512, 526, 524, 502, 543, 502, 535, 511, 516, 512, 454, 507, 504, 519, 556, 512, 487, 518, 517, 526, 528, 515, 509, 531, 507, 568, 528, 508, 506, 484, 517, 527, 515, 519, 510, 479, 480, 534, 506] payload_rows_per_dst=[10990, 10826, 10876, 11110, 10834, 10912, 10976, 11054] dup_rows_per_dst=[5382, 5394, 5498, 5402, 5340, 5688, 5406, 5384] combine_rows_per_src=[10990, 10826, 10876, 11110, 10834, 10912, 10976, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87608 payload_rows_remote=76632 dup_rows=43464 dup_rows_remote=38020 dedup_ratio=0.331604 router_tokens_per_expert=[485, 471, 510, 500, 513, 498, 552, 511, 513, 514, 513, 548, 500, 476, 520, 497, 491, 511, 497, 481, 512, 531, 503, 509, 528, 496, 509, 498, 515, 512, 480, 497, 493, 529, 515, 478, 497, 513, 473, 544, 516, 534, 511, 499, 526, 517, 546, 521, 515, 553, 514, 522, 513, 508, 506, 497, 487, 527, 512, 544, 510, 516, 549, 488, 505, 504, 469, 509, 508, 528, 540, 484, 522, 503, 501, 518, 522, 564, 512, 539, 503, 523, 528, 534, 487, 500, 497, 525, 531, 516, 517, 532, 491, 475, 499, 552, 490, 538, 526, 536, 529, 496, 519, 503, 504, 524, 529, 505, 523, 522, 513, 499, 524, 475, 497, 499, 561, 505, 473, 498, 514, 524, 524, 518, 509, 532, 521, 504] payload_rows_per_dst=[10886, 10928, 10952, 11028, 10926, 10976, 11004, 10908] dup_rows_per_dst=[5356, 5212, 5472, 5494, 5530, 5444, 5508, 5448] combine_rows_per_src=[10886, 10928, 10952, 11028, 10926, 10976, 11004, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87494 payload_rows_remote=76512 dup_rows=43578 dup_rows_remote=38088 dedup_ratio=0.332474 router_tokens_per_expert=[542, 493, 489, 502, 478, 548, 499, 484, 491, 519, 491, 540, 478, 506, 524, 507, 519, 535, 490, 504, 521, 518, 494, 508, 477, 523, 499, 492, 550, 444, 488, 490, 502, 526, 541, 507, 523, 528, 493, 568, 541, 522, 493, 510, 525, 488, 523, 518, 524, 532, 567, 518, 526, 506, 529, 472, 488, 537, 557, 487, 528, 535, 506, 476, 533, 502, 551, 537, 541, 486, 484, 536, 499, 554, 514, 483, 521, 511, 521, 488, 509, 527, 510, 504, 471, 504, 486, 537, 502, 520, 533, 505, 500, 538, 551, 528, 474, 500, 516, 473, 489, 500, 504, 516, 544, 516, 550, 525, 502, 521, 508, 553, 524, 517, 486, 509, 522, 510, 506, 511, 521, 504, 479, 494, 504, 522, 513, 498] payload_rows_per_dst=[10790, 10820, 11020, 11014, 10924, 10960, 10984, 10982] dup_rows_per_dst=[5392, 5284, 5596, 5562, 5366, 5490, 5398, 5490] combine_rows_per_src=[10790, 10820, 11020, 11014, 10924, 10960, 10984, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=624525312 dispatch_meta_bytes=457368 combine_payload_bytes=624525312 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624982680 ingress_bytes=700022784 link_bytes=700022784 token_bytes=1249507992 weight_movement_bytes=75497472 total_nvlink_bytes=1325005464 weight_to_token_ratio=0.060422 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.877274258e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76492 dup_rows=43582 dup_rows_remote=38176 dedup_ratio=0.332504 router_tokens_per_expert=[496, 531, 537, 493, 514, 514, 511, 531, 495, 498, 510, 495, 521, 499, 540, 542, 498, 528, 489, 491, 510, 512, 497, 515, 487, 522, 520, 498, 516, 542, 499, 508, 507, 545, 516, 500, 553, 501, 496, 523, 504, 512, 524, 483, 510, 479, 511, 547, 528, 489, 495, 517, 542, 507, 489, 527, 526, 507, 480, 528, 476, 486, 553, 522, 521, 519, 478, 499, 483, 519, 467, 537, 526, 511, 571, 506, 529, 551, 516, 534, 516, 498, 480, 552, 526, 533, 512, 530, 520, 496, 543, 484, 524, 519, 494, 520, 503, 517, 518, 535, 505, 500, 517, 504, 517, 524, 530, 512, 502, 515, 502, 501, 519, 486, 483, 533, 481, 515, 524, 501, 466, 507, 512, 511, 515, 526, 514, 485] payload_rows_per_dst=[10976, 10852, 10956, 10918, 10954, 10958, 10998, 10878] dup_rows_per_dst=[5478, 5412, 5466, 5426, 5580, 5536, 5406, 5278] combine_rows_per_src=[10976, 10852, 10956, 10918, 10954, 10958, 10998, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628342784 dispatch_meta_bytes=458792 combine_payload_bytes=628342784 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628801576 ingress_bytes=703840256 link_bytes=703840256 token_bytes=1257144360 weight_movement_bytes=75497472 total_nvlink_bytes=1332641832 weight_to_token_ratio=0.060055 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.925685176e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87322 payload_rows_remote=76462 dup_rows=43750 dup_rows_remote=38070 dedup_ratio=0.333786 router_tokens_per_expert=[497, 519, 534, 522, 486, 504, 485, 492, 518, 510, 542, 527, 544, 474, 497, 518, 525, 521, 509, 550, 494, 505, 522, 508, 512, 525, 502, 554, 510, 537, 523, 517, 516, 505, 488, 497, 528, 495, 492, 521, 526, 515, 506, 504, 513, 494, 521, 518, 514, 532, 522, 522, 521, 528, 543, 505, 518, 532, 499, 498, 503, 510, 522, 501, 523, 516, 488, 521, 512, 528, 514, 459, 519, 518, 481, 514, 516, 547, 526, 523, 475, 504, 540, 463, 501, 480, 514, 512, 530, 498, 489, 530, 525, 478, 520, 483, 527, 508, 498, 521, 549, 493, 470, 490, 536, 533, 503, 532, 487, 483, 493, 498, 535, 513, 529, 523, 550, 539, 533, 464, 567, 525, 505, 480, 483, 464, 533, 533] payload_rows_per_dst=[10894, 11026, 10874, 10860, 10990, 10732, 10918, 11028] dup_rows_per_dst=[5444, 5602, 5404, 5680, 5420, 5352, 5324, 5524] combine_rows_per_src=[10894, 11026, 10874, 10860, 10990, 10732, 10918, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=629850112 dispatch_meta_bytes=459616 combine_payload_bytes=629850112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630309728 ingress_bytes=705347584 link_bytes=705347584 token_bytes=1260159840 weight_movement_bytes=75497472 total_nvlink_bytes=1335657312 weight_to_token_ratio=0.059911 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.944800217e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=458608 combine_payload_bytes=627769344 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=628227952 ingress_bytes=854261760 link_bytes=854261760 token_bytes=1255997296 weight_movement_bytes=226492416 total_nvlink_bytes=1482489712 weight_to_token_ratio=0.180329 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083324158e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626786304 dispatch_meta_bytes=458400 combine_payload_bytes=626786304 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627244704 ingress_bytes=702283776 link_bytes=702283776 token_bytes=1254031008 weight_movement_bytes=75497472 total_nvlink_bytes=1329528480 weight_to_token_ratio=0.060204 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905946819e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87650 payload_rows_remote=76686 dup_rows=43422 dup_rows_remote=37874 dedup_ratio=0.331284 router_tokens_per_expert=[497, 517, 498, 508, 550, 505, 525, 540, 499, 510, 487, 513, 534, 529, 522, 509, 496, 495, 487, 566, 519, 476, 504, 491, 511, 472, 493, 535, 501, 525, 537, 517, 522, 528, 483, 506, 495, 509, 513, 495, 507, 505, 490, 475, 497, 509, 474, 527, 515, 495, 498, 514, 519, 539, 483, 502, 547, 521, 495, 524, 507, 500, 534, 531, 502, 514, 516, 509, 504, 523, 553, 492, 507, 500, 525, 553, 538, 518, 505, 497, 527, 529, 541, 506, 530, 516, 486, 522, 520, 480, 503, 505, 556, 533, 485, 515, 507, 525, 530, 515, 531, 548, 532, 538, 541, 489, 507, 506, 519, 489, 505, 490, 561, 473, 499, 497, 499, 519, 532, 490, 477, 500, 511, 537, 519, 515, 488, 510] payload_rows_per_dst=[10938, 11000, 10792, 11008, 10964, 10984, 11004, 10960] dup_rows_per_dst=[5548, 5250, 5278, 5440, 5548, 5524, 5540, 5294] combine_rows_per_src=[10938, 11000, 10792, 11008, 10964, 10984, 11004, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626622464 dispatch_meta_bytes=458672 combine_payload_bytes=626622464 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627081136 ingress_bytes=853114880 link_bytes=853114880 token_bytes=1253703600 weight_movement_bytes=226492416 total_nvlink_bytes=1480196016 weight_to_token_ratio=0.180659 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081869753e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626376704 dispatch_meta_bytes=458128 combine_payload_bytes=626376704 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626834832 ingress_bytes=777371648 link_bytes=777371648 token_bytes=1253211536 weight_movement_bytes=150994944 total_nvlink_bytes=1404206480 weight_to_token_ratio=0.120486 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.858166730e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=628211712 dispatch_meta_bytes=458240 combine_payload_bytes=628211712 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628669952 ingress_bytes=779206656 link_bytes=779206656 token_bytes=1256881664 weight_movement_bytes=150994944 total_nvlink_bytes=1407876608 weight_to_token_ratio=0.120135 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.881437214e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76418 dup_rows=43590 dup_rows_remote=38076 dedup_ratio=0.332565 router_tokens_per_expert=[526, 533, 498, 520, 515, 507, 466, 510, 521, 496, 500, 531, 535, 499, 545, 536, 512, 457, 510, 528, 497, 496, 510, 489, 493, 532, 539, 526, 549, 474, 514, 518, 521, 528, 455, 530, 516, 538, 481, 520, 492, 496, 507, 517, 516, 516, 502, 529, 510, 514, 509, 566, 535, 519, 528, 495, 491, 513, 518, 491, 508, 492, 505, 512, 500, 518, 525, 534, 508, 521, 508, 543, 480, 535, 505, 505, 487, 510, 496, 475, 497, 527, 510, 519, 538, 506, 527, 520, 479, 480, 558, 527, 489, 507, 481, 552, 507, 505, 549, 531, 520, 501, 525, 477, 493, 488, 511, 521, 524, 518, 507, 531, 484, 491, 535, 526, 516, 518, 536, 524, 499, 512, 483, 517, 494, 510, 511, 553] payload_rows_per_dst=[11064, 11236, 10880, 10604, 10936, 10870, 10890, 11002] dup_rows_per_dst=[5514, 6062, 5466, 5100, 5366, 5248, 5418, 5416] combine_rows_per_src=[11064, 11236, 10880, 10604, 10936, 10870, 10890, 11002] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87396 payload_rows_remote=76352 dup_rows=43676 dup_rows_remote=38160 dedup_ratio=0.333221 router_tokens_per_expert=[522, 510, 488, 539, 537, 509, 503, 472, 520, 533, 506, 509, 497, 497, 497, 529, 516, 536, 501, 511, 541, 502, 516, 512, 515, 564, 491, 509, 505, 527, 504, 490, 520, 490, 501, 561, 535, 508, 523, 518, 519, 535, 523, 534, 487, 497, 517, 512, 505, 536, 504, 543, 499, 505, 494, 487, 518, 501, 471, 484, 521, 527, 498, 483, 525, 524, 493, 502, 502, 530, 502, 487, 501, 507, 508, 517, 530, 460, 511, 525, 496, 512, 508, 511, 478, 538, 535, 521, 512, 520, 488, 528, 505, 518, 522, 458, 547, 520, 527, 504, 559, 526, 484, 519, 493, 454, 522, 553, 492, 501, 508, 517, 516, 480, 532, 538, 505, 520, 513, 518, 512, 493, 517, 528, 505, 538, 522, 535] payload_rows_per_dst=[10968, 10994, 11044, 10720, 10842, 10882, 10960, 10986] dup_rows_per_dst=[5368, 5486, 5516, 5432, 5406, 5418, 5492, 5558] combine_rows_per_src=[10968, 10994, 11044, 10720, 10842, 10882, 10960, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87416 payload_rows_remote=76482 dup_rows=43656 dup_rows_remote=38244 dedup_ratio=0.333069 router_tokens_per_expert=[503, 506, 484, 517, 521, 522, 484, 472, 478, 442, 544, 495, 532, 473, 521, 522, 509, 493, 500, 536, 523, 500, 541, 484, 469, 500, 505, 541, 566, 523, 481, 502, 586, 531, 555, 518, 474, 519, 498, 531, 492, 516, 513, 526, 543, 485, 524, 491, 488, 524, 487, 507, 476, 511, 523, 551, 521, 533, 476, 515, 494, 520, 498, 485, 513, 526, 472, 540, 507, 555, 517, 486, 510, 485, 490, 503, 536, 518, 500, 524, 517, 499, 538, 500, 499, 513, 511, 525, 509, 471, 534, 536, 529, 556, 497, 547, 555, 533, 542, 511, 521, 554, 510, 493, 547, 468, 454, 552, 532, 516, 534, 508, 528, 507, 501, 505, 517, 510, 517, 507, 549, 526, 477, 447, 525, 502, 500, 525] payload_rows_per_dst=[10790, 10934, 10938, 10902, 10976, 10930, 11020, 10926] dup_rows_per_dst=[5242, 5412, 5666, 5316, 5388, 5632, 5640, 5360] combine_rows_per_src=[10790, 10934, 10938, 10902, 10976, 10930, 11020, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87468 payload_rows_remote=76522 dup_rows=43604 dup_rows_remote=37944 dedup_ratio=0.332672 router_tokens_per_expert=[524, 496, 525, 515, 489, 524, 495, 497, 502, 548, 508, 518, 527, 500, 524, 493, 528, 458, 516, 509, 515, 483, 533, 512, 485, 531, 515, 578, 483, 510, 473, 504, 490, 530, 497, 492, 534, 535, 492, 505, 530, 509, 518, 543, 521, 527, 515, 539, 503, 507, 524, 532, 530, 506, 512, 532, 513, 537, 537, 483, 535, 512, 523, 495, 525, 514, 539, 476, 536, 517, 521, 525, 524, 509, 521, 530, 486, 494, 467, 507, 529, 546, 544, 522, 517, 550, 487, 482, 514, 524, 518, 523, 513, 506, 500, 528, 521, 492, 505, 517, 531, 475, 503, 547, 520, 518, 510, 540, 544, 496, 478, 478, 505, 535, 511, 487, 481, 525, 491, 478, 510, 498, 499, 497, 484, 509, 495, 486] payload_rows_per_dst=[10990, 10908, 11024, 10982, 10888, 10946, 10942, 10788] dup_rows_per_dst=[5380, 5358, 5530, 5580, 5494, 5660, 5408, 5194] combine_rows_per_src=[10990, 10908, 11024, 10982, 10888, 10946, 10942, 10788] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87102 payload_rows_remote=76142 dup_rows=43970 dup_rows_remote=38476 dedup_ratio=0.335464 router_tokens_per_expert=[536, 503, 486, 507, 509, 529, 488, 525, 502, 504, 515, 527, 494, 512, 475, 481, 513, 521, 551, 520, 472, 518, 573, 510, 528, 518, 521, 479, 486, 520, 514, 526, 486, 506, 492, 528, 483, 525, 507, 531, 580, 491, 498, 502, 533, 512, 527, 507, 515, 529, 532, 499, 503, 495, 541, 493, 508, 496, 503, 510, 468, 503, 467, 524, 545, 579, 507, 508, 503, 516, 495, 490, 482, 524, 523, 562, 539, 502, 487, 541, 537, 481, 495, 556, 505, 510, 544, 531, 520, 528, 515, 456, 516, 503, 494, 513, 486, 525, 531, 540, 547, 524, 494, 485, 535, 516, 472, 543, 488, 560, 508, 484, 520, 515, 510, 473, 537, 491, 490, 491, 544, 527, 493, 525, 512, 511, 479, 516] payload_rows_per_dst=[10800, 10786, 10772, 10990, 10894, 10826, 11074, 10960] dup_rows_per_dst=[5386, 5216, 5364, 5678, 5634, 5474, 5724, 5494] combine_rows_per_src=[10800, 10786, 10772, 10990, 10894, 10826, 11074, 10960] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626016256 dispatch_meta_bytes=457976 combine_payload_bytes=626016256 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626474232 ingress_bytes=701513728 link_bytes=701513728 token_bytes=1252490488 weight_movement_bytes=75497472 total_nvlink_bytes=1327987960 weight_to_token_ratio=0.060278 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.896181526e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87380 payload_rows_remote=76508 dup_rows=43692 dup_rows_remote=38394 dedup_ratio=0.333344 router_tokens_per_expert=[507, 503, 539, 504, 498, 534, 538, 511, 544, 515, 503, 514, 531, 548, 496, 535, 509, 525, 484, 515, 514, 511, 512, 526, 493, 557, 491, 478, 513, 576, 519, 527, 536, 483, 538, 526, 541, 513, 513, 503, 514, 506, 502, 483, 500, 482, 488, 488, 521, 529, 523, 537, 496, 505, 534, 538, 479, 500, 501, 494, 541, 475, 505, 478, 499, 532, 522, 469, 512, 522, 487, 527, 503, 507, 528, 515, 545, 531, 546, 462, 477, 507, 506, 503, 521, 529, 510, 487, 492, 486, 522, 509, 524, 542, 519, 514, 497, 559, 498, 515, 479, 502, 501, 496, 509, 493, 500, 493, 515, 510, 518, 500, 484, 524, 494, 508, 529, 496, 520, 501, 524, 531, 515, 531, 560, 506, 485, 546] payload_rows_per_dst=[11024, 11016, 10836, 10796, 10978, 10902, 10872, 10956] dup_rows_per_dst=[5616, 5484, 5396, 5516, 5436, 5394, 5298, 5552] combine_rows_per_src=[11024, 11016, 10836, 10796, 10978, 10902, 10872, 10956] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625475584 dispatch_meta_bytes=458048 combine_payload_bytes=625475584 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625933632 ingress_bytes=776470528 link_bytes=776470528 token_bytes=1251409216 weight_movement_bytes=150994944 total_nvlink_bytes=1402404160 weight_to_token_ratio=0.120660 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.846739260e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87466 payload_rows_remote=76542 dup_rows=43606 dup_rows_remote=38180 dedup_ratio=0.332687 router_tokens_per_expert=[485, 483, 521, 505, 496, 521, 520, 514, 496, 486, 510, 538, 480, 525, 512, 517, 537, 521, 488, 514, 538, 487, 528, 473, 508, 513, 515, 520, 505, 517, 539, 520, 519, 484, 493, 493, 547, 525, 500, 534, 475, 550, 521, 530, 496, 531, 496, 492, 528, 547, 465, 489, 533, 472, 522, 526, 502, 514, 502, 532, 523, 497, 511, 512, 490, 539, 497, 511, 506, 519, 511, 522, 531, 463, 522, 528, 524, 500, 484, 487, 479, 516, 492, 470, 526, 511, 537, 543, 540, 536, 490, 508, 528, 527, 518, 481, 501, 497, 536, 512, 493, 502, 488, 509, 493, 508, 540, 539, 514, 479, 534, 512, 525, 512, 553, 513, 551, 524, 516, 489, 523, 503, 541, 538, 542, 493, 503, 524] payload_rows_per_dst=[10852, 10972, 10872, 10924, 10778, 10988, 10988, 11092] dup_rows_per_dst=[5366, 5474, 5500, 5426, 5490, 5416, 5326, 5608] combine_rows_per_src=[10852, 10972, 10872, 10924, 10778, 10988, 10988, 11092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76564 dup_rows=43584 dup_rows_remote=38160 dedup_ratio=0.332520 router_tokens_per_expert=[515, 523, 520, 508, 530, 523, 493, 495, 531, 495, 510, 494, 499, 507, 522, 513, 497, 543, 557, 501, 535, 510, 536, 525, 510, 526, 496, 477, 535, 573, 531, 523, 505, 524, 568, 492, 502, 500, 545, 515, 507, 488, 492, 516, 482, 520, 505, 483, 507, 521, 518, 504, 498, 503, 501, 483, 512, 491, 566, 507, 509, 527, 494, 525, 509, 493, 493, 534, 503, 533, 526, 514, 473, 499, 487, 554, 520, 504, 549, 483, 481, 542, 560, 526, 484, 514, 537, 528, 502, 511, 515, 503, 496, 532, 521, 490, 488, 510, 460, 504, 489, 481, 487, 507, 498, 495, 498, 556, 505, 527, 536, 511, 509, 480, 505, 535, 519, 497, 512, 506, 501, 473, 506, 525, 532, 506, 539, 560] payload_rows_per_dst=[10864, 11044, 10880, 10958, 10924, 11040, 10796, 10982] dup_rows_per_dst=[5492, 5706, 5408, 5374, 5424, 5444, 5308, 5428] combine_rows_per_src=[10864, 11044, 10880, 10958, 10924, 11040, 10796, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626540544 dispatch_meta_bytes=458904 combine_payload_bytes=626540544 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626999448 ingress_bytes=853032960 link_bytes=853032960 token_bytes=1253539992 weight_movement_bytes=226492416 total_nvlink_bytes=1480032408 weight_to_token_ratio=0.180682 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081765867e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=623755264 dispatch_meta_bytes=458472 combine_payload_bytes=623755264 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624213736 ingress_bytes=699252736 link_bytes=699252736 token_bytes=1247969000 weight_movement_bytes=75497472 total_nvlink_bytes=1323466472 weight_to_token_ratio=0.060496 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.867508965e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626868224 dispatch_meta_bytes=457864 combine_payload_bytes=626868224 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627326088 ingress_bytes=853360640 link_bytes=853360640 token_bytes=1254194312 weight_movement_bytes=226492416 total_nvlink_bytes=1480686728 weight_to_token_ratio=0.180588 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082181411e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=459608 combine_payload_bytes=626753536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627213144 ingress_bytes=702251008 link_bytes=702251008 token_bytes=1253966680 weight_movement_bytes=75497472 total_nvlink_bytes=1329464152 weight_to_token_ratio=0.060207 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905531274e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627032064 dispatch_meta_bytes=458888 combine_payload_bytes=627032064 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627490952 ingress_bytes=702529536 link_bytes=702529536 token_bytes=1254523016 weight_movement_bytes=75497472 total_nvlink_bytes=1330020488 weight_to_token_ratio=0.060180 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909063402e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627212288 dispatch_meta_bytes=458896 combine_payload_bytes=627212288 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627671184 ingress_bytes=778207232 link_bytes=778207232 token_bytes=1254883472 weight_movement_bytes=150994944 total_nvlink_bytes=1405878416 weight_to_token_ratio=0.120326 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.868763111e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87560 payload_rows_remote=76400 dup_rows=43512 dup_rows_remote=37738 dedup_ratio=0.331970 router_tokens_per_expert=[562, 497, 519, 513, 493, 537, 530, 530, 485, 503, 534, 518, 540, 515, 487, 495, 497, 486, 498, 512, 497, 502, 529, 517, 516, 476, 500, 514, 536, 502, 556, 497, 547, 538, 517, 472, 515, 514, 509, 504, 508, 505, 520, 546, 486, 479, 488, 515, 492, 537, 515, 535, 501, 477, 514, 494, 500, 532, 521, 526, 499, 514, 551, 531, 526, 527, 547, 479, 524, 543, 510, 513, 497, 507, 530, 512, 544, 488, 497, 522, 492, 485, 486, 501, 534, 486, 541, 566, 544, 505, 557, 563, 496, 537, 492, 484, 538, 512, 472, 486, 489, 545, 541, 512, 501, 503, 495, 490, 467, 517, 556, 488, 500, 540, 525, 484, 512, 487, 524, 537, 522, 473, 456, 492, 515, 494, 545, 488] payload_rows_per_dst=[11160, 10926, 10916, 10970, 11024, 10774, 10924, 10866] dup_rows_per_dst=[5774, 5356, 5410, 5508, 5496, 5346, 5300, 5322] combine_rows_per_src=[11160, 10926, 10916, 10970, 11024, 10774, 10924, 10866] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87490 payload_rows_remote=76594 dup_rows=43582 dup_rows_remote=38098 dedup_ratio=0.332504 router_tokens_per_expert=[543, 483, 478, 558, 516, 506, 501, 505, 481, 507, 494, 520, 529, 503, 478, 506, 534, 489, 503, 529, 529, 516, 539, 493, 508, 506, 538, 510, 511, 522, 548, 494, 475, 536, 498, 528, 492, 487, 545, 515, 551, 520, 498, 538, 467, 539, 497, 504, 491, 515, 488, 535, 503, 488, 523, 476, 508, 571, 499, 492, 522, 519, 502, 528, 499, 476, 524, 537, 500, 495, 538, 543, 511, 496, 552, 515, 514, 514, 508, 498, 507, 528, 533, 536, 530, 513, 503, 476, 553, 492, 507, 526, 517, 524, 476, 523, 531, 526, 474, 517, 495, 495, 520, 525, 548, 519, 534, 469, 545, 487, 491, 516, 459, 492, 565, 522, 517, 459, 511, 544, 525, 490, 480, 566, 497, 515, 508, 503] payload_rows_per_dst=[10926, 11002, 10896, 10928, 10974, 10994, 10918, 10852] dup_rows_per_dst=[5290, 5536, 5484, 5392, 5466, 5494, 5466, 5454] combine_rows_per_src=[10926, 11002, 10896, 10928, 10974, 10994, 10918, 10852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87660 payload_rows_remote=76746 dup_rows=43412 dup_rows_remote=38016 dedup_ratio=0.331207 router_tokens_per_expert=[528, 532, 521, 567, 530, 522, 561, 540, 532, 493, 521, 509, 520, 510, 554, 515, 542, 511, 463, 492, 494, 493, 535, 505, 532, 534, 509, 520, 491, 491, 528, 515, 543, 488, 514, 508, 482, 498, 482, 550, 474, 503, 496, 506, 501, 473, 503, 476, 512, 539, 472, 476, 559, 546, 503, 490, 495, 506, 503, 500, 473, 520, 507, 483, 513, 513, 523, 523, 518, 499, 502, 495, 494, 480, 504, 479, 484, 533, 525, 496, 489, 513, 477, 561, 528, 513, 472, 530, 524, 536, 519, 519, 508, 560, 519, 516, 545, 550, 520, 533, 536, 513, 500, 509, 502, 493, 524, 529, 513, 498, 486, 516, 523, 506, 550, 518, 523, 487, 523, 496, 499, 518, 504, 502, 533, 494, 499, 538] payload_rows_per_dst=[11200, 10914, 10812, 10804, 10906, 10956, 11090, 10978] dup_rows_per_dst=[5710, 5396, 5182, 5364, 5256, 5612, 5444, 5448] combine_rows_per_src=[11200, 10914, 10812, 10804, 10906, 10956, 11090, 10978] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76472 dup_rows=43584 dup_rows_remote=38024 dedup_ratio=0.332520 router_tokens_per_expert=[491, 532, 526, 512, 502, 541, 462, 556, 527, 485, 522, 522, 511, 537, 515, 478, 526, 538, 530, 539, 533, 492, 526, 516, 527, 490, 469, 498, 543, 504, 525, 521, 498, 487, 499, 509, 514, 582, 541, 499, 521, 501, 499, 497, 467, 471, 542, 548, 531, 527, 549, 508, 512, 477, 539, 542, 488, 530, 518, 513, 507, 488, 460, 495, 463, 518, 516, 538, 507, 509, 489, 521, 524, 526, 506, 491, 525, 490, 565, 532, 511, 519, 492, 520, 530, 562, 491, 524, 498, 513, 494, 496, 506, 475, 476, 488, 533, 531, 496, 512, 468, 482, 498, 525, 509, 512, 494, 507, 518, 501, 498, 498, 529, 541, 528, 508, 506, 526, 528, 569, 499, 525, 480, 493, 525, 509, 502, 516] payload_rows_per_dst=[10852, 10954, 11088, 10940, 10910, 10572, 11156, 11016] dup_rows_per_dst=[5204, 5370, 5874, 5414, 5536, 4958, 5668, 5560] combine_rows_per_src=[10852, 10954, 11088, 10940, 10910, 10572, 11156, 11016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87814 payload_rows_remote=76824 dup_rows=43258 dup_rows_remote=37780 dedup_ratio=0.330032 router_tokens_per_expert=[521, 528, 494, 499, 541, 501, 509, 523, 514, 506, 523, 556, 476, 497, 518, 531, 570, 499, 530, 505, 493, 533, 516, 496, 504, 500, 479, 542, 526, 496, 491, 514, 503, 518, 524, 517, 527, 499, 499, 535, 480, 543, 472, 500, 506, 499, 483, 495, 530, 550, 519, 511, 543, 518, 519, 512, 543, 556, 463, 519, 566, 511, 545, 456, 518, 495, 552, 529, 534, 503, 483, 499, 497, 525, 485, 526, 500, 514, 540, 551, 526, 519, 494, 524, 538, 524, 490, 506, 512, 479, 510, 506, 529, 533, 524, 520, 498, 503, 492, 485, 510, 491, 486, 478, 507, 532, 511, 467, 534, 472, 510, 480, 499, 551, 497, 485, 521, 517, 534, 525, 508, 524, 490, 523, 489, 560, 489, 491] payload_rows_per_dst=[11026, 10888, 10864, 11236, 11074, 10990, 10710, 11026] dup_rows_per_dst=[5448, 5500, 5336, 5486, 5428, 5478, 5202, 5380] combine_rows_per_src=[11026, 10888, 10864, 11236, 11074, 10990, 10710, 11026] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625868800 dispatch_meta_bytes=456552 combine_payload_bytes=625868800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626325352 ingress_bytes=701366272 link_bytes=701366272 token_bytes=1252194152 weight_movement_bytes=75497472 total_nvlink_bytes=1327691624 weight_to_token_ratio=0.060292 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894311577e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87678 payload_rows_remote=76644 dup_rows=43394 dup_rows_remote=37850 dedup_ratio=0.331070 router_tokens_per_expert=[472, 513, 541, 514, 506, 519, 503, 525, 531, 508, 496, 512, 499, 522, 501, 510, 494, 499, 490, 495, 512, 518, 506, 500, 493, 518, 519, 525, 511, 522, 495, 507, 474, 500, 520, 556, 503, 518, 504, 478, 538, 482, 533, 499, 509, 547, 486, 506, 507, 509, 506, 506, 522, 539, 529, 511, 510, 507, 524, 545, 529, 544, 521, 516, 458, 500, 501, 549, 496, 528, 494, 551, 500, 526, 515, 520, 502, 531, 515, 535, 519, 487, 490, 541, 522, 532, 490, 494, 526, 494, 520, 501, 542, 504, 485, 502, 513, 530, 520, 524, 504, 525, 528, 529, 519, 489, 493, 545, 527, 513, 530, 500, 482, 524, 486, 550, 540, 514, 490, 525, 504, 538, 457, 453, 494, 534, 521, 511] payload_rows_per_dst=[10888, 10842, 11004, 11090, 10982, 10930, 11034, 10908] dup_rows_per_dst=[5456, 5366, 5302, 5560, 5460, 5368, 5544, 5338] combine_rows_per_src=[10888, 10842, 11004, 11090, 10982, 10930, 11034, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627458048 dispatch_meta_bytes=458768 combine_payload_bytes=627458048 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627916816 ingress_bytes=702955520 link_bytes=702955520 token_bytes=1255374864 weight_movement_bytes=75497472 total_nvlink_bytes=1330872336 weight_to_token_ratio=0.060139 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914465478e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87560 payload_rows_remote=76744 dup_rows=43512 dup_rows_remote=38092 dedup_ratio=0.331970 router_tokens_per_expert=[509, 518, 549, 476, 547, 493, 503, 530, 467, 491, 520, 533, 521, 508, 479, 495, 518, 525, 527, 483, 531, 474, 536, 541, 506, 540, 519, 530, 509, 524, 529, 494, 505, 481, 490, 498, 494, 543, 543, 525, 524, 500, 498, 515, 515, 501, 482, 542, 521, 503, 531, 500, 479, 537, 499, 511, 503, 491, 530, 520, 489, 506, 500, 498, 512, 511, 527, 474, 515, 487, 522, 520, 489, 520, 518, 495, 504, 528, 491, 528, 512, 532, 543, 495, 538, 532, 526, 493, 532, 513, 550, 496, 525, 522, 571, 496, 548, 519, 534, 511, 514, 542, 491, 514, 516, 516, 505, 461, 509, 496, 519, 494, 495, 506, 539, 520, 529, 485, 501, 515, 510, 496, 467, 516, 522, 511, 486, 533] payload_rows_per_dst=[10950, 10986, 10912, 10816, 10896, 11152, 10946, 10902] dup_rows_per_dst=[5328, 5586, 5400, 5420, 5386, 5600, 5432, 5360] combine_rows_per_src=[10950, 10986, 10912, 10816, 10896, 11152, 10946, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87532 payload_rows_remote=76604 dup_rows=43540 dup_rows_remote=38172 dedup_ratio=0.332184 router_tokens_per_expert=[534, 510, 487, 532, 521, 535, 497, 515, 483, 486, 467, 510, 513, 501, 498, 522, 542, 482, 525, 505, 530, 542, 500, 508, 513, 519, 530, 469, 535, 515, 512, 517, 475, 539, 509, 512, 547, 530, 495, 511, 499, 518, 500, 551, 506, 477, 501, 498, 509, 506, 523, 501, 506, 533, 511, 519, 504, 509, 535, 494, 519, 551, 508, 484, 481, 478, 502, 507, 472, 501, 490, 515, 537, 526, 538, 511, 473, 529, 539, 549, 548, 513, 518, 554, 516, 561, 578, 512, 520, 493, 489, 510, 510, 493, 532, 536, 499, 524, 523, 480, 502, 475, 508, 500, 515, 476, 507, 527, 530, 527, 496, 490, 495, 492, 535, 485, 517, 479, 512, 477, 514, 508, 516, 542, 524, 555, 515, 525] payload_rows_per_dst=[10938, 11020, 10926, 10966, 10928, 11066, 10858, 10830] dup_rows_per_dst=[5284, 5468, 5410, 5458, 5368, 5700, 5300, 5552] combine_rows_per_src=[10938, 11020, 10926, 10966, 10928, 11066, 10858, 10830] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628703232 dispatch_meta_bytes=459048 combine_payload_bytes=628703232 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629162280 ingress_bytes=779698176 link_bytes=779698176 token_bytes=1257865512 weight_movement_bytes=150994944 total_nvlink_bytes=1408860456 weight_to_token_ratio=0.120041 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.887670380e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626458624 dispatch_meta_bytes=457984 combine_payload_bytes=626458624 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626916608 ingress_bytes=777453568 link_bytes=777453568 token_bytes=1253375232 weight_movement_bytes=150994944 total_nvlink_bytes=1404370176 weight_to_token_ratio=0.120471 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.859205591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=629342208 dispatch_meta_bytes=458416 combine_payload_bytes=629342208 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=629800624 ingress_bytes=855834624 link_bytes=855834624 token_bytes=1259142832 weight_movement_bytes=226492416 total_nvlink_bytes=1485635248 weight_to_token_ratio=0.179878 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085318771e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627867648 dispatch_meta_bytes=457976 combine_payload_bytes=627867648 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628325624 ingress_bytes=703365120 link_bytes=703365120 token_bytes=1256193272 weight_movement_bytes=75497472 total_nvlink_bytes=1331690744 weight_to_token_ratio=0.060100 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919659783e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=628686848 dispatch_meta_bytes=459344 combine_payload_bytes=628686848 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=629146192 ingress_bytes=779681792 link_bytes=779681792 token_bytes=1257833040 weight_movement_bytes=150994944 total_nvlink_bytes=1408827984 weight_to_token_ratio=0.120044 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.887462608e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=459104 combine_payload_bytes=627539968 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627999072 ingress_bytes=703037440 link_bytes=703037440 token_bytes=1255539040 weight_movement_bytes=75497472 total_nvlink_bytes=1331036512 weight_to_token_ratio=0.060132 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.915504339e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87600 payload_rows_remote=76522 dup_rows=43472 dup_rows_remote=37978 dedup_ratio=0.331665 router_tokens_per_expert=[544, 517, 507, 495, 552, 514, 504, 510, 510, 509, 505, 534, 492, 491, 504, 506, 507, 487, 502, 512, 500, 501, 511, 507, 510, 464, 500, 534, 509, 539, 501, 538, 523, 537, 571, 489, 506, 508, 482, 535, 526, 481, 555, 510, 511, 561, 530, 466, 524, 520, 536, 493, 504, 531, 501, 513, 513, 500, 480, 507, 476, 489, 522, 490, 537, 510, 510, 501, 482, 516, 522, 502, 498, 528, 531, 518, 543, 510, 496, 547, 513, 485, 503, 508, 514, 500, 543, 502, 528, 529, 485, 496, 533, 479, 522, 515, 532, 525, 505, 519, 479, 531, 485, 544, 477, 467, 482, 525, 510, 519, 513, 539, 522, 516, 520, 509, 512, 494, 527, 513, 519, 524, 526, 476, 555, 529, 518, 512] payload_rows_per_dst=[11078, 10844, 11060, 10872, 10964, 10870, 10960, 10952] dup_rows_per_dst=[5494, 5400, 5522, 5326, 5538, 5332, 5344, 5516] combine_rows_per_src=[11078, 10844, 11060, 10872, 10964, 10870, 10960, 10952] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87440 payload_rows_remote=76460 dup_rows=43632 dup_rows_remote=38202 dedup_ratio=0.332886 router_tokens_per_expert=[506, 529, 527, 532, 517, 488, 487, 568, 511, 520, 513, 540, 515, 527, 507, 485, 506, 510, 502, 470, 521, 520, 533, 514, 497, 530, 488, 488, 487, 529, 511, 451, 504, 517, 491, 508, 468, 541, 520, 562, 499, 534, 549, 480, 513, 506, 525, 488, 516, 507, 556, 497, 497, 527, 521, 474, 546, 517, 510, 480, 514, 507, 514, 537, 511, 541, 533, 482, 535, 487, 507, 519, 536, 526, 521, 499, 482, 507, 522, 477, 524, 484, 492, 495, 517, 510, 521, 558, 515, 502, 536, 521, 510, 502, 514, 527, 515, 466, 509, 496, 548, 533, 496, 509, 527, 518, 505, 489, 483, 521, 547, 508, 558, 526, 506, 514, 499, 506, 507, 521, 510, 533, 500, 494, 505, 529, 495, 496] payload_rows_per_dst=[11056, 10806, 10980, 10972, 10870, 10984, 10842, 10930] dup_rows_per_dst=[5488, 5308, 5430, 5468, 5500, 5472, 5498, 5468] combine_rows_per_src=[11056, 10806, 10980, 10972, 10870, 10984, 10842, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87294 payload_rows_remote=76260 dup_rows=43778 dup_rows_remote=38252 dedup_ratio=0.334000 router_tokens_per_expert=[488, 505, 527, 485, 526, 517, 536, 513, 490, 508, 533, 486, 517, 545, 485, 510, 495, 538, 541, 488, 488, 533, 559, 535, 527, 551, 501, 546, 470, 510, 479, 519, 531, 527, 566, 513, 504, 504, 528, 502, 460, 535, 501, 530, 486, 497, 537, 500, 511, 532, 455, 509, 530, 537, 524, 512, 492, 537, 502, 543, 518, 468, 461, 542, 513, 484, 493, 538, 516, 494, 514, 491, 503, 530, 517, 533, 501, 499, 548, 548, 523, 517, 509, 525, 494, 482, 470, 513, 515, 519, 502, 560, 515, 512, 511, 520, 499, 527, 514, 488, 489, 488, 508, 505, 496, 505, 518, 579, 529, 508, 457, 518, 508, 488, 525, 508, 508, 498, 522, 544, 507, 499, 453, 506, 517, 522, 522, 527] payload_rows_per_dst=[10932, 11034, 10834, 10894, 11002, 10944, 10746, 10908] dup_rows_per_dst=[5410, 5526, 5608, 5452, 5442, 5430, 5510, 5400] combine_rows_per_src=[10932, 11034, 10834, 10894, 11002, 10944, 10746, 10908] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87200 payload_rows_remote=76296 dup_rows=43872 dup_rows_remote=38344 dedup_ratio=0.334717 router_tokens_per_expert=[525, 526, 498, 549, 497, 534, 531, 490, 537, 517, 503, 532, 538, 492, 488, 534, 519, 510, 512, 465, 558, 517, 521, 519, 527, 501, 458, 508, 530, 510, 510, 514, 507, 496, 517, 517, 505, 522, 492, 519, 501, 547, 513, 534, 513, 495, 542, 516, 512, 514, 518, 541, 488, 523, 515, 497, 475, 521, 492, 513, 538, 492, 525, 510, 496, 487, 500, 525, 494, 541, 515, 523, 518, 468, 481, 491, 500, 502, 496, 497, 556, 537, 510, 528, 493, 520, 498, 469, 512, 534, 538, 490, 502, 499, 494, 510, 520, 494, 520, 518, 505, 532, 507, 509, 495, 495, 500, 529, 525, 534, 502, 532, 507, 524, 495, 492, 490, 529, 565, 490, 544, 499, 528, 502, 513, 539, 483, 515] payload_rows_per_dst=[10562, 11136, 10568, 10894, 11210, 10862, 11064, 10904] dup_rows_per_dst=[5114, 5674, 4824, 5454, 5942, 5518, 5818, 5528] combine_rows_per_src=[10562, 11136, 10568, 10894, 11210, 10862, 11064, 10904] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626868224 dispatch_meta_bytes=458000 combine_payload_bytes=626868224 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627326224 ingress_bytes=777863168 link_bytes=777863168 token_bytes=1254194448 weight_movement_bytes=150994944 total_nvlink_bytes=1405189392 weight_to_token_ratio=0.120392 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.864399895e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87580 payload_rows_remote=76452 dup_rows=43492 dup_rows_remote=38082 dedup_ratio=0.331818 router_tokens_per_expert=[527, 531, 493, 495, 476, 566, 554, 496, 523, 520, 511, 476, 518, 508, 498, 520, 513, 507, 498, 510, 509, 511, 515, 482, 542, 513, 510, 492, 520, 516, 521, 529, 494, 492, 563, 500, 500, 531, 521, 542, 498, 553, 522, 488, 542, 512, 495, 536, 544, 524, 487, 505, 503, 531, 533, 482, 506, 503, 521, 505, 502, 450, 507, 522, 492, 493, 528, 478, 518, 536, 522, 529, 495, 499, 512, 486, 519, 475, 523, 504, 508, 495, 540, 538, 539, 516, 530, 492, 518, 492, 524, 511, 489, 518, 527, 532, 473, 462, 487, 471, 509, 497, 496, 531, 497, 475, 520, 490, 557, 537, 505, 516, 542, 518, 520, 524, 518, 468, 521, 599, 539, 519, 534, 512, 521, 512, 486, 488] payload_rows_per_dst=[10908, 10902, 11030, 10850, 10872, 11128, 10808, 11082] dup_rows_per_dst=[5516, 5474, 5548, 5400, 5346, 5410, 5238, 5560] combine_rows_per_src=[10908, 10902, 11030, 10850, 10872, 11128, 10808, 11082] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87388 payload_rows_remote=76504 dup_rows=43684 dup_rows_remote=38268 dedup_ratio=0.333282 router_tokens_per_expert=[464, 530, 511, 537, 533, 540, 473, 553, 583, 533, 497, 503, 526, 537, 507, 527, 504, 511, 520, 504, 551, 516, 490, 527, 519, 529, 514, 474, 496, 470, 519, 514, 531, 504, 497, 492, 536, 527, 510, 457, 509, 529, 516, 490, 506, 504, 534, 552, 511, 481, 499, 529, 513, 553, 489, 532, 511, 495, 456, 466, 542, 506, 519, 541, 509, 496, 515, 503, 511, 499, 518, 491, 497, 536, 488, 498, 513, 516, 483, 501, 504, 505, 512, 487, 547, 515, 497, 506, 535, 495, 495, 511, 505, 537, 541, 512, 515, 504, 492, 467, 519, 481, 533, 507, 505, 505, 525, 531, 519, 529, 511, 507, 497, 530, 504, 487, 528, 517, 529, 516, 503, 527, 518, 499, 518, 536, 501, 549] payload_rows_per_dst=[11092, 10858, 10888, 10896, 10912, 10910, 10884, 10948] dup_rows_per_dst=[5616, 5458, 5428, 5390, 5308, 5498, 5416, 5570] combine_rows_per_src=[11092, 10858, 10888, 10896, 10912, 10910, 10884, 10948] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626360320 dispatch_meta_bytes=458648 combine_payload_bytes=626360320 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626818968 ingress_bytes=777355264 link_bytes=777355264 token_bytes=1253179288 weight_movement_bytes=150994944 total_nvlink_bytes=1404174232 weight_to_token_ratio=0.120489 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.857958958e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87638 payload_rows_remote=76862 dup_rows=43434 dup_rows_remote=38128 dedup_ratio=0.331375 router_tokens_per_expert=[540, 591, 502, 528, 521, 482, 548, 506, 486, 546, 529, 527, 507, 499, 535, 509, 543, 503, 521, 515, 540, 481, 570, 494, 541, 506, 486, 463, 493, 512, 480, 507, 543, 515, 493, 536, 505, 532, 495, 537, 485, 478, 505, 484, 496, 543, 501, 528, 537, 560, 494, 513, 471, 562, 530, 518, 490, 477, 517, 493, 471, 511, 530, 485, 509, 497, 488, 506, 496, 475, 508, 521, 507, 517, 476, 511, 536, 497, 503, 494, 523, 466, 491, 560, 506, 480, 507, 507, 527, 541, 548, 536, 546, 479, 498, 515, 498, 512, 515, 462, 519, 505, 502, 515, 525, 536, 534, 495, 537, 528, 507, 527, 537, 533, 486, 491, 528, 513, 500, 514, 503, 499, 526, 498, 475, 571, 498, 530] payload_rows_per_dst=[11130, 10906, 10986, 10924, 10776, 11036, 10984, 10896] dup_rows_per_dst=[5582, 5404, 5366, 5394, 5306, 5424, 5450, 5508] combine_rows_per_src=[11130, 10906, 10986, 10924, 10776, 11036, 10984, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87302 payload_rows_remote=76250 dup_rows=43770 dup_rows_remote=38206 dedup_ratio=0.333939 router_tokens_per_expert=[544, 516, 531, 510, 505, 509, 511, 519, 525, 489, 494, 501, 501, 533, 545, 515, 502, 513, 501, 496, 524, 523, 546, 539, 484, 509, 521, 496, 526, 496, 536, 471, 564, 534, 558, 525, 548, 548, 482, 521, 509, 538, 509, 491, 489, 471, 554, 525, 529, 493, 503, 514, 537, 518, 520, 517, 485, 513, 535, 547, 513, 523, 537, 524, 536, 535, 510, 509, 445, 518, 522, 486, 541, 550, 515, 504, 523, 496, 476, 497, 459, 478, 509, 483, 531, 490, 529, 468, 508, 513, 478, 538, 473, 472, 519, 511, 553, 513, 493, 515, 528, 498, 533, 504, 486, 551, 521, 498, 481, 497, 515, 506, 554, 520, 505, 480, 509, 483, 451, 535, 495, 537, 548, 467, 510, 544, 482, 497] payload_rows_per_dst=[10978, 10896, 11052, 11052, 10860, 10734, 10860, 10870] dup_rows_per_dst=[5518, 5470, 5680, 5564, 5466, 5184, 5524, 5364] combine_rows_per_src=[10978, 10896, 11052, 11052, 10860, 10734, 10860, 10870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=624721920 dispatch_meta_bytes=458048 combine_payload_bytes=624721920 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625179968 ingress_bytes=700219392 link_bytes=700219392 token_bytes=1249901888 weight_movement_bytes=75497472 total_nvlink_bytes=1325399360 weight_to_token_ratio=0.060403 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.879767524e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625016832 dispatch_meta_bytes=458560 combine_payload_bytes=625016832 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625475392 ingress_bytes=700514304 link_bytes=700514304 token_bytes=1250492224 weight_movement_bytes=75497472 total_nvlink_bytes=1325989696 weight_to_token_ratio=0.060374 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.883507423e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626294784 dispatch_meta_bytes=458136 combine_payload_bytes=626294784 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=626752920 ingress_bytes=701792256 link_bytes=701792256 token_bytes=1253047704 weight_movement_bytes=75497472 total_nvlink_bytes=1328545176 weight_to_token_ratio=0.060251 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899713653e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=626720768 dispatch_meta_bytes=459088 combine_payload_bytes=626720768 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627179856 ingress_bytes=702218240 link_bytes=702218240 token_bytes=1253900624 weight_movement_bytes=75497472 total_nvlink_bytes=1329398096 weight_to_token_ratio=0.060210 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.905115730e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629653504 dispatch_meta_bytes=459960 combine_payload_bytes=629653504 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630113464 ingress_bytes=705150976 link_bytes=705150976 token_bytes=1259766968 weight_movement_bytes=75497472 total_nvlink_bytes=1335264440 weight_to_token_ratio=0.059930 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.942306951e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=624640000 dispatch_meta_bytes=457824 combine_payload_bytes=624640000 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625097824 ingress_bytes=700137472 link_bytes=700137472 token_bytes=1249737824 weight_movement_bytes=75497472 total_nvlink_bytes=1325235296 weight_to_token_ratio=0.060411 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.878728663e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87690 payload_rows_remote=76796 dup_rows=43382 dup_rows_remote=38106 dedup_ratio=0.330978 router_tokens_per_expert=[515, 492, 516, 510, 514, 494, 511, 520, 486, 536, 525, 482, 478, 489, 523, 494, 498, 492, 511, 519, 523, 483, 495, 530, 529, 500, 534, 490, 475, 515, 494, 522, 533, 515, 563, 465, 505, 538, 534, 485, 519, 507, 571, 536, 539, 497, 573, 545, 530, 553, 518, 543, 491, 507, 494, 483, 529, 518, 507, 485, 531, 520, 506, 492, 531, 510, 499, 452, 537, 486, 489, 537, 543, 511, 528, 476, 524, 545, 496, 466, 527, 502, 478, 534, 538, 523, 521, 514, 525, 514, 506, 521, 495, 558, 511, 529, 498, 512, 514, 490, 527, 532, 528, 510, 492, 515, 539, 480, 492, 522, 521, 543, 488, 469, 518, 480, 523, 533, 489, 514, 517, 497, 475, 492, 511, 521, 521, 520] payload_rows_per_dst=[10894, 10952, 11052, 10996, 10910, 11112, 10942, 10832] dup_rows_per_dst=[5276, 5268, 5798, 5472, 5350, 5426, 5488, 5304] combine_rows_per_src=[10894, 10952, 11052, 10996, 10910, 11112, 10942, 10832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87592 payload_rows_remote=76704 dup_rows=43480 dup_rows_remote=38002 dedup_ratio=0.331726 router_tokens_per_expert=[553, 513, 510, 528, 492, 546, 492, 519, 489, 520, 492, 554, 496, 490, 524, 555, 475, 512, 510, 542, 512, 505, 534, 491, 526, 510, 529, 501, 492, 519, 525, 523, 538, 499, 500, 543, 505, 495, 501, 503, 544, 505, 492, 504, 506, 506, 542, 500, 526, 523, 508, 516, 501, 453, 528, 500, 516, 525, 534, 499, 496, 519, 446, 490, 522, 481, 480, 520, 504, 505, 528, 550, 543, 560, 513, 524, 533, 527, 515, 526, 515, 486, 534, 504, 496, 518, 467, 512, 539, 528, 500, 459, 449, 514, 507, 505, 529, 529, 539, 518, 489, 505, 528, 508, 508, 503, 523, 485, 524, 523, 488, 521, 501, 548, 531, 492, 505, 509, 512, 520, 559, 521, 502, 516, 514, 505, 488, 487] payload_rows_per_dst=[11044, 10896, 10888, 10850, 11106, 10798, 11048, 10962] dup_rows_per_dst=[5502, 5516, 5478, 5310, 5556, 5268, 5392, 5458] combine_rows_per_src=[11044, 10896, 10888, 10850, 11106, 10798, 11048, 10962] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 1, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87568 payload_rows_remote=76562 dup_rows=43504 dup_rows_remote=37884 dedup_ratio=0.331909 router_tokens_per_expert=[524, 538, 534, 535, 505, 507, 472, 523, 489, 467, 501, 509, 509, 519, 505, 513, 541, 512, 537, 542, 520, 511, 490, 499, 507, 547, 496, 549, 511, 540, 487, 524, 528, 494, 558, 515, 530, 525, 534, 509, 525, 526, 470, 500, 536, 498, 527, 480, 518, 534, 495, 536, 488, 494, 523, 545, 510, 505, 498, 515, 487, 528, 580, 534, 489, 520, 502, 515, 542, 521, 481, 562, 534, 514, 500, 481, 491, 507, 505, 505, 502, 514, 529, 536, 496, 502, 511, 488, 511, 462, 518, 506, 510, 504, 486, 497, 505, 497, 523, 503, 501, 498, 524, 523, 527, 533, 504, 504, 502, 511, 496, 519, 494, 506, 505, 517, 520, 522, 514, 523, 504, 488, 539, 530, 441, 497, 502, 515] payload_rows_per_dst=[10906, 11006, 11006, 11004, 10922, 10834, 10964, 10926] dup_rows_per_dst=[5394, 5620, 5504, 5576, 5416, 5310, 5376, 5308] combine_rows_per_src=[10906, 11006, 11006, 11004, 10922, 10834, 10964, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87536 payload_rows_remote=76444 dup_rows=43536 dup_rows_remote=37970 dedup_ratio=0.332153 router_tokens_per_expert=[529, 535, 533, 545, 537, 524, 488, 485, 496, 539, 495, 541, 470, 510, 468, 551, 501, 533, 526, 537, 498, 536, 506, 477, 528, 467, 461, 496, 469, 482, 517, 544, 519, 506, 485, 509, 551, 508, 539, 491, 508, 509, 505, 490, 546, 494, 522, 474, 509, 487, 501, 505, 517, 523, 518, 515, 498, 512, 500, 519, 528, 509, 488, 540, 529, 545, 516, 473, 502, 482, 513, 515, 499, 510, 538, 523, 521, 503, 465, 542, 531, 503, 535, 495, 515, 546, 522, 510, 524, 486, 521, 475, 504, 530, 501, 529, 500, 508, 494, 523, 497, 518, 568, 495, 546, 500, 536, 563, 483, 511, 511, 527, 536, 466, 530, 498, 547, 547, 471, 515, 512, 516, 522, 484, 496, 491, 529, 544] payload_rows_per_dst=[11044, 10718, 10854, 10846, 10888, 11206, 10888, 11092] dup_rows_per_dst=[5380, 4912, 5374, 5492, 5470, 5802, 5540, 5566] combine_rows_per_src=[11044, 10718, 10854, 10846, 10888, 11206, 10888, 11092] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629112832 dispatch_meta_bytes=459608 combine_payload_bytes=629112832 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=629572440 ingress_bytes=931102720 link_bytes=931102720 token_bytes=1258685272 weight_movement_bytes=301989888 total_nvlink_bytes=1560675160 weight_to_token_ratio=0.239925 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.180769312e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76436 dup_rows=43648 dup_rows_remote=38306 dedup_ratio=0.333008 router_tokens_per_expert=[537, 489, 518, 534, 507, 530, 538, 489, 510, 538, 544, 496, 527, 487, 476, 519, 529, 505, 479, 530, 528, 516, 548, 480, 479, 532, 490, 513, 528, 494, 549, 521, 551, 486, 519, 521, 495, 493, 529, 519, 520, 544, 504, 501, 491, 486, 511, 525, 515, 510, 515, 531, 484, 487, 519, 532, 512, 505, 507, 514, 538, 538, 532, 528, 500, 518, 482, 479, 498, 471, 522, 521, 499, 494, 496, 505, 520, 498, 515, 510, 503, 514, 525, 507, 512, 509, 570, 504, 484, 513, 506, 480, 512, 540, 486, 500, 507, 516, 563, 530, 511, 519, 503, 505, 500, 522, 490, 507, 503, 542, 534, 514, 497, 533, 524, 472, 520, 541, 498, 496, 492, 499, 516, 470, 556, 533, 503, 505] payload_rows_per_dst=[10960, 10950, 10820, 10958, 10874, 10988, 11018, 10856] dup_rows_per_dst=[5518, 5492, 5570, 5576, 5182, 5342, 5514, 5454] combine_rows_per_src=[10960, 10950, 10820, 10958, 10874, 10988, 11018, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87400 payload_rows_remote=76454 dup_rows=43672 dup_rows_remote=38188 dedup_ratio=0.333191 router_tokens_per_expert=[499, 503, 517, 468, 535, 530, 543, 484, 496, 503, 527, 514, 535, 514, 504, 492, 537, 539, 494, 456, 519, 526, 498, 502, 524, 552, 524, 505, 531, 516, 497, 555, 496, 516, 513, 497, 480, 531, 534, 481, 496, 495, 516, 502, 530, 513, 517, 503, 518, 467, 525, 487, 505, 492, 529, 535, 500, 500, 494, 527, 490, 530, 501, 536, 496, 487, 478, 514, 535, 501, 506, 507, 493, 528, 561, 515, 481, 503, 519, 510, 474, 551, 496, 517, 501, 521, 500, 533, 483, 549, 513, 503, 528, 526, 515, 492, 518, 503, 481, 526, 503, 532, 507, 522, 510, 531, 491, 494, 556, 523, 491, 527, 523, 507, 498, 511, 508, 517, 486, 510, 500, 536, 527, 552, 492, 530, 533, 560] payload_rows_per_dst=[10922, 10954, 10874, 10818, 10944, 10992, 10946, 10950] dup_rows_per_dst=[5406, 5596, 5366, 5454, 5324, 5412, 5484, 5630] combine_rows_per_src=[10922, 10954, 10874, 10818, 10944, 10992, 10946, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87522 payload_rows_remote=76640 dup_rows=43550 dup_rows_remote=38116 dedup_ratio=0.332260 router_tokens_per_expert=[489, 477, 488, 512, 512, 529, 528, 528, 484, 506, 505, 513, 571, 501, 514, 529, 510, 530, 525, 500, 481, 514, 500, 559, 494, 540, 528, 509, 506, 539, 533, 550, 506, 516, 482, 513, 497, 523, 538, 511, 537, 501, 539, 479, 547, 520, 496, 530, 473, 494, 485, 498, 486, 480, 513, 550, 505, 480, 539, 536, 524, 522, 519, 511, 509, 481, 521, 515, 526, 474, 497, 562, 497, 509, 551, 544, 494, 513, 482, 483, 505, 469, 483, 557, 541, 512, 546, 493, 478, 566, 477, 520, 517, 537, 530, 479, 523, 502, 481, 512, 467, 505, 525, 538, 462, 543, 511, 522, 509, 523, 508, 468, 498, 524, 553, 540, 500, 520, 511, 522, 539, 501, 471, 489, 511, 488, 524, 524] payload_rows_per_dst=[11008, 11042, 10952, 10882, 10882, 10990, 10844, 10922] dup_rows_per_dst=[5364, 5594, 5518, 5348, 5434, 5430, 5354, 5508] combine_rows_per_src=[11008, 11042, 10952, 10882, 10882, 10990, 10844, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87646 payload_rows_remote=76690 dup_rows=43426 dup_rows_remote=38106 dedup_ratio=0.331314 router_tokens_per_expert=[502, 516, 518, 538, 509, 518, 471, 529, 487, 510, 507, 539, 529, 490, 562, 534, 543, 525, 537, 495, 524, 525, 510, 519, 480, 532, 517, 531, 468, 495, 551, 524, 496, 545, 554, 540, 560, 503, 510, 492, 517, 557, 519, 511, 503, 529, 493, 495, 535, 512, 583, 496, 504, 526, 482, 530, 483, 526, 467, 504, 509, 522, 497, 462, 543, 514, 506, 528, 511, 544, 531, 513, 467, 530, 504, 479, 522, 523, 507, 561, 473, 516, 532, 512, 498, 529, 527, 506, 522, 495, 479, 520, 503, 474, 495, 527, 498, 547, 481, 494, 524, 538, 484, 510, 516, 503, 509, 481, 479, 467, 507, 527, 480, 506, 475, 517, 515, 485, 491, 509, 513, 522, 542, 490, 505, 519, 486, 528] payload_rows_per_dst=[11066, 10952, 11092, 10956, 11058, 10830, 10882, 10810] dup_rows_per_dst=[5452, 5600, 5556, 5320, 5508, 5386, 5248, 5356] combine_rows_per_src=[11066, 10952, 11092, 10956, 11058, 10830, 10882, 10810] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628359168 dispatch_meta_bytes=458824 combine_payload_bytes=628359168 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628817992 ingress_bytes=779354112 link_bytes=779354112 token_bytes=1257177160 weight_movement_bytes=150994944 total_nvlink_bytes=1408172104 weight_to_token_ratio=0.120106 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.883307164e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627195904 dispatch_meta_bytes=457784 combine_payload_bytes=627195904 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627653688 ingress_bytes=778190848 link_bytes=778190848 token_bytes=1254849592 weight_movement_bytes=150994944 total_nvlink_bytes=1405844536 weight_to_token_ratio=0.120329 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.868555339e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626229248 dispatch_meta_bytes=457656 combine_payload_bytes=626229248 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626686904 ingress_bytes=701726720 link_bytes=701726720 token_bytes=1252916152 weight_movement_bytes=75497472 total_nvlink_bytes=1328413624 weight_to_token_ratio=0.060257 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898882565e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626163712 dispatch_meta_bytes=458968 combine_payload_bytes=626163712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626622680 ingress_bytes=701661184 link_bytes=701661184 token_bytes=1252786392 weight_movement_bytes=75497472 total_nvlink_bytes=1328283864 weight_to_token_ratio=0.060264 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898051476e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626311168 dispatch_meta_bytes=458568 combine_payload_bytes=626311168 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626769736 ingress_bytes=777306112 link_bytes=777306112 token_bytes=1253080904 weight_movement_bytes=150994944 total_nvlink_bytes=1404075848 weight_to_token_ratio=0.120499 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.857335641e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627834880 dispatch_meta_bytes=459024 combine_payload_bytes=627834880 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628293904 ingress_bytes=778829824 link_bytes=778829824 token_bytes=1256128784 weight_movement_bytes=150994944 total_nvlink_bytes=1407123728 weight_to_token_ratio=0.120207 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876658454e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=628244480 dispatch_meta_bytes=459184 combine_payload_bytes=628244480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628703664 ingress_bytes=703741952 link_bytes=703741952 token_bytes=1256948144 weight_movement_bytes=75497472 total_nvlink_bytes=1332445616 weight_to_token_ratio=0.060064 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.924438543e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87676 payload_rows_remote=76888 dup_rows=43396 dup_rows_remote=38078 dedup_ratio=0.331085 router_tokens_per_expert=[467, 519, 490, 496, 492, 504, 509, 518, 508, 488, 485, 522, 499, 545, 480, 531, 503, 510, 518, 526, 527, 504, 528, 549, 482, 517, 482, 514, 458, 479, 522, 493, 531, 519, 531, 508, 523, 480, 537, 527, 541, 462, 471, 494, 528, 465, 535, 458, 529, 530, 491, 507, 578, 513, 501, 501, 520, 533, 473, 485, 522, 497, 514, 521, 512, 542, 501, 560, 530, 501, 518, 522, 519, 512, 521, 541, 548, 538, 509, 506, 516, 511, 502, 516, 524, 477, 530, 518, 500, 514, 524, 499, 551, 554, 497, 509, 505, 518, 554, 506, 551, 524, 483, 512, 520, 527, 490, 510, 509, 543, 520, 515, 516, 502, 486, 498, 531, 489, 546, 492, 512, 510, 534, 468, 528, 472, 519, 534] payload_rows_per_dst=[10788, 10962, 10848, 11068, 11094, 11064, 10938, 10914] dup_rows_per_dst=[5318, 5262, 5372, 5614, 5414, 5420, 5636, 5360] combine_rows_per_src=[10788, 10962, 10848, 11068, 11094, 11064, 10938, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87318 payload_rows_remote=76418 dup_rows=43754 dup_rows_remote=38414 dedup_ratio=0.333817 router_tokens_per_expert=[480, 540, 520, 517, 507, 530, 527, 491, 488, 517, 529, 480, 522, 533, 520, 538, 518, 520, 510, 526, 507, 517, 505, 552, 540, 526, 435, 522, 497, 542, 493, 540, 514, 491, 508, 510, 522, 505, 518, 504, 497, 515, 545, 534, 487, 470, 501, 499, 491, 498, 541, 495, 542, 509, 552, 499, 500, 476, 543, 552, 550, 500, 515, 513, 519, 505, 509, 559, 495, 490, 473, 504, 515, 522, 525, 532, 514, 499, 517, 510, 484, 499, 532, 532, 529, 532, 487, 520, 488, 469, 500, 510, 534, 497, 517, 510, 523, 501, 499, 538, 489, 513, 519, 534, 526, 520, 500, 538, 488, 487, 528, 518, 505, 515, 525, 518, 501, 518, 515, 495, 493, 568, 505, 458, 489, 487, 503, 507] payload_rows_per_dst=[10878, 10948, 10900, 10940, 10840, 10920, 11020, 10872] dup_rows_per_dst=[5600, 5552, 5340, 5612, 5536, 5360, 5422, 5332] combine_rows_per_src=[10878, 10948, 10900, 10940, 10840, 10920, 11020, 10872] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=86944 payload_rows_remote=76138 dup_rows=44128 dup_rows_remote=38466 dedup_ratio=0.336670 router_tokens_per_expert=[524, 523, 515, 506, 513, 497, 464, 501, 537, 554, 523, 529, 471, 502, 528, 535, 508, 529, 530, 529, 481, 536, 523, 496, 505, 513, 503, 510, 537, 493, 521, 522, 476, 500, 586, 483, 485, 474, 542, 517, 506, 459, 529, 486, 527, 537, 546, 490, 504, 503, 497, 503, 521, 487, 546, 504, 514, 559, 508, 492, 517, 506, 478, 536, 487, 494, 539, 522, 522, 524, 531, 528, 503, 483, 546, 494, 518, 535, 478, 522, 522, 514, 520, 521, 512, 538, 493, 543, 520, 553, 490, 502, 528, 504, 544, 498, 522, 508, 489, 493, 449, 509, 480, 482, 497, 503, 538, 509, 528, 491, 522, 489, 534, 519, 491, 530, 522, 521, 511, 515, 516, 510, 488, 515, 484, 516, 546, 505] payload_rows_per_dst=[10884, 10482, 11006, 10866, 10992, 10956, 10952, 10806] dup_rows_per_dst=[5560, 5012, 5698, 5300, 5622, 5648, 5626, 5662] combine_rows_per_src=[10884, 10482, 11006, 10866, 10992, 10956, 10952, 10806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76370 dup_rows=43584 dup_rows_remote=37974 dedup_ratio=0.332520 router_tokens_per_expert=[509, 495, 506, 496, 536, 517, 533, 516, 507, 502, 489, 523, 496, 524, 534, 517, 494, 530, 524, 505, 500, 528, 502, 544, 503, 540, 502, 569, 538, 518, 540, 527, 530, 520, 515, 490, 528, 466, 511, 498, 540, 482, 541, 504, 503, 527, 516, 532, 529, 539, 511, 481, 525, 513, 500, 515, 508, 501, 495, 495, 499, 499, 525, 520, 504, 529, 491, 466, 508, 486, 502, 530, 536, 491, 461, 504, 540, 515, 496, 537, 493, 491, 546, 514, 480, 515, 536, 503, 500, 482, 482, 532, 499, 500, 520, 524, 487, 562, 517, 530, 479, 526, 529, 505, 531, 484, 524, 493, 507, 483, 495, 535, 473, 544, 514, 541, 514, 498, 499, 541, 481, 498, 518, 505, 565, 522, 496, 505] payload_rows_per_dst=[11008, 11118, 10874, 10930, 10728, 10878, 10948, 11004] dup_rows_per_dst=[5392, 5610, 5532, 5380, 5464, 5356, 5426, 5424] combine_rows_per_src=[11008, 11118, 10874, 10930, 10728, 10878, 10948, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=629866496 dispatch_meta_bytes=459864 combine_payload_bytes=629866496 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=630326360 ingress_bytes=856358912 link_bytes=856358912 token_bytes=1260192856 weight_movement_bytes=226492416 total_nvlink_bytes=1486685272 weight_to_token_ratio=0.179728 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085983642e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87434 payload_rows_remote=76604 dup_rows=43638 dup_rows_remote=38162 dedup_ratio=0.332932 router_tokens_per_expert=[484, 538, 516, 564, 467, 537, 492, 519, 518, 451, 505, 530, 474, 472, 513, 519, 525, 501, 498, 509, 525, 463, 495, 487, 491, 530, 494, 514, 516, 503, 542, 499, 507, 515, 536, 538, 499, 497, 528, 503, 511, 499, 519, 524, 492, 495, 497, 490, 494, 533, 510, 521, 519, 523, 522, 495, 499, 537, 512, 543, 516, 522, 521, 531, 517, 541, 505, 536, 510, 493, 524, 502, 473, 497, 535, 539, 500, 499, 531, 489, 518, 544, 511, 509, 546, 504, 526, 517, 526, 461, 507, 502, 494, 514, 480, 494, 521, 544, 525, 510, 519, 496, 528, 552, 581, 532, 486, 502, 508, 474, 500, 529, 482, 499, 525, 528, 526, 545, 526, 517, 504, 503, 506, 537, 515, 520, 495, 518] payload_rows_per_dst=[10952, 10820, 10940, 11048, 10866, 10830, 11060, 10918] dup_rows_per_dst=[5246, 5364, 5360, 5548, 5516, 5476, 5554, 5574] combine_rows_per_src=[10952, 10820, 10940, 11048, 10866, 10830, 11060, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87332 payload_rows_remote=76424 dup_rows=43740 dup_rows_remote=38368 dedup_ratio=0.333710 router_tokens_per_expert=[527, 485, 544, 497, 534, 534, 517, 529, 506, 534, 553, 505, 488, 528, 479, 495, 517, 475, 544, 524, 548, 548, 529, 492, 543, 488, 500, 525, 497, 514, 544, 567, 494, 496, 500, 499, 517, 493, 513, 511, 475, 508, 448, 543, 528, 514, 537, 500, 536, 498, 509, 503, 545, 497, 513, 485, 514, 500, 519, 468, 490, 488, 508, 494, 520, 537, 522, 513, 480, 503, 477, 530, 519, 500, 490, 508, 506, 528, 494, 513, 519, 547, 547, 525, 491, 540, 501, 518, 506, 500, 498, 512, 502, 519, 476, 530, 488, 530, 529, 504, 498, 477, 512, 496, 528, 532, 544, 532, 520, 532, 533, 516, 480, 514, 521, 507, 539, 523, 521, 488, 544, 488, 519, 472, 520, 492, 518, 495] payload_rows_per_dst=[10956, 11032, 10876, 10728, 10908, 11004, 11026, 10802] dup_rows_per_dst=[5554, 5678, 5276, 5406, 5372, 5458, 5516, 5480] combine_rows_per_src=[10956, 11032, 10876, 10728, 10908, 11004, 11026, 10802] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87530 payload_rows_remote=76830 dup_rows=43542 dup_rows_remote=38318 dedup_ratio=0.332199 router_tokens_per_expert=[492, 533, 484, 555, 491, 481, 480, 479, 525, 501, 497, 554, 517, 536, 513, 533, 499, 495, 489, 530, 545, 540, 505, 503, 501, 509, 505, 537, 515, 526, 496, 527, 482, 459, 516, 498, 489, 544, 515, 514, 554, 500, 500, 527, 502, 504, 522, 513, 530, 501, 543, 535, 543, 520, 525, 527, 480, 493, 510, 546, 496, 471, 489, 504, 505, 477, 503, 538, 480, 516, 525, 568, 516, 510, 540, 535, 484, 529, 533, 503, 546, 523, 530, 511, 563, 464, 512, 568, 506, 544, 549, 508, 466, 501, 517, 487, 518, 534, 530, 509, 482, 488, 479, 485, 490, 470, 514, 517, 488, 507, 465, 486, 514, 525, 495, 518, 518, 479, 558, 523, 541, 527, 471, 472, 541, 538, 521, 531] payload_rows_per_dst=[10998, 11034, 10930, 10944, 10932, 10972, 10700, 11020] dup_rows_per_dst=[5344, 5410, 5348, 5482, 5592, 5618, 5224, 5524] combine_rows_per_src=[10998, 11034, 10930, 10944, 10932, 10972, 10700, 11020] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87574 payload_rows_remote=76526 dup_rows=43498 dup_rows_remote=38072 dedup_ratio=0.331863 router_tokens_per_expert=[507, 503, 535, 480, 523, 541, 514, 487, 506, 500, 531, 497, 492, 496, 493, 492, 562, 476, 494, 542, 483, 536, 547, 523, 520, 503, 518, 556, 469, 505, 548, 489, 545, 489, 505, 540, 508, 567, 518, 505, 512, 480, 512, 486, 534, 541, 489, 503, 526, 542, 514, 498, 523, 508, 499, 541, 489, 540, 497, 488, 543, 540, 473, 516, 520, 522, 487, 508, 499, 527, 510, 488, 513, 517, 496, 480, 526, 540, 552, 528, 526, 550, 510, 512, 486, 491, 511, 483, 522, 507, 515, 521, 524, 508, 518, 532, 522, 518, 494, 492, 537, 470, 515, 515, 518, 512, 502, 490, 499, 538, 515, 551, 503, 502, 491, 503, 526, 503, 503, 495, 502, 499, 520, 516, 450, 514, 515, 538] payload_rows_per_dst=[10892, 10964, 10948, 11048, 10946, 10950, 10988, 10838] dup_rows_per_dst=[5302, 5578, 5520, 5426, 5480, 5482, 5388, 5322] combine_rows_per_src=[10892, 10964, 10948, 11048, 10946, 10950, 10988, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626016256 dispatch_meta_bytes=459328 combine_payload_bytes=626016256 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626475584 ingress_bytes=701513728 link_bytes=701513728 token_bytes=1252491840 weight_movement_bytes=75497472 total_nvlink_bytes=1327989312 weight_to_token_ratio=0.060278 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.896181526e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=623722496 dispatch_meta_bytes=458416 combine_payload_bytes=623722496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624180912 ingress_bytes=699219968 link_bytes=699219968 token_bytes=1247903408 weight_movement_bytes=75497472 total_nvlink_bytes=1323400880 weight_to_token_ratio=0.060499 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.867093421e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625623040 dispatch_meta_bytes=457376 combine_payload_bytes=625623040 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626080416 ingress_bytes=852115456 link_bytes=852115456 token_bytes=1251703456 weight_movement_bytes=226492416 total_nvlink_bytes=1478195872 weight_to_token_ratio=0.180947 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080602343e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627539968 dispatch_meta_bytes=459064 combine_payload_bytes=627539968 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627999032 ingress_bytes=778534912 link_bytes=778534912 token_bytes=1255539000 weight_movement_bytes=150994944 total_nvlink_bytes=1406533944 weight_to_token_ratio=0.120263 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872918555e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=459168 combine_payload_bytes=626065408 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626524576 ingress_bytes=701562880 link_bytes=701562880 token_bytes=1252589984 weight_movement_bytes=75497472 total_nvlink_bytes=1328087456 weight_to_token_ratio=0.060273 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.896804843e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=629391360 dispatch_meta_bytes=460592 combine_payload_bytes=629391360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629851952 ingress_bytes=704888832 link_bytes=704888832 token_bytes=1259243312 weight_movement_bytes=75497472 total_nvlink_bytes=1334740784 weight_to_token_ratio=0.059955 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.938982596e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626900992 dispatch_meta_bytes=458392 combine_payload_bytes=626900992 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=627359384 ingress_bytes=928890880 link_bytes=928890880 token_bytes=1254260376 weight_movement_bytes=301989888 total_nvlink_bytes=1556250264 weight_to_token_ratio=0.240771 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177964387e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87346 payload_rows_remote=76306 dup_rows=43726 dup_rows_remote=38260 dedup_ratio=0.333603 router_tokens_per_expert=[512, 531, 505, 521, 492, 534, 506, 473, 490, 493, 546, 515, 537, 514, 536, 509, 510, 496, 528, 529, 517, 489, 474, 551, 535, 497, 536, 528, 479, 480, 524, 523, 498, 532, 553, 578, 462, 515, 523, 506, 523, 481, 532, 514, 496, 529, 485, 535, 537, 492, 513, 522, 518, 503, 486, 528, 498, 524, 527, 519, 532, 534, 508, 500, 470, 489, 529, 521, 502, 532, 522, 496, 513, 525, 499, 513, 514, 500, 502, 512, 491, 512, 502, 538, 498, 532, 522, 520, 481, 512, 474, 495, 500, 530, 504, 515, 529, 530, 482, 510, 484, 531, 521, 482, 490, 504, 479, 496, 524, 526, 500, 523, 512, 530, 476, 480, 559, 545, 473, 505, 547, 515, 508, 541, 550, 490, 509, 507] payload_rows_per_dst=[11040, 10928, 10942, 10912, 10870, 10938, 10716, 11000] dup_rows_per_dst=[5466, 5542, 5582, 5502, 5408, 5226, 5506, 5494] combine_rows_per_src=[11040, 10928, 10942, 10912, 10870, 10938, 10716, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625098752 dispatch_meta_bytes=458264 combine_payload_bytes=625098752 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625557016 ingress_bytes=700596224 link_bytes=700596224 token_bytes=1250655768 weight_movement_bytes=75497472 total_nvlink_bytes=1326153240 weight_to_token_ratio=0.060366 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.884546284e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87556 payload_rows_remote=76678 dup_rows=43516 dup_rows_remote=38122 dedup_ratio=0.332001 router_tokens_per_expert=[547, 512, 484, 527, 525, 540, 499, 497, 497, 505, 515, 476, 476, 518, 545, 517, 502, 525, 503, 504, 500, 553, 506, 496, 500, 519, 506, 551, 529, 549, 525, 503, 518, 522, 505, 516, 484, 557, 531, 512, 518, 465, 537, 488, 517, 480, 487, 499, 502, 530, 512, 540, 512, 531, 498, 512, 533, 499, 481, 539, 486, 524, 513, 494, 552, 502, 552, 561, 498, 523, 496, 512, 545, 507, 500, 495, 493, 510, 513, 461, 484, 497, 531, 483, 529, 475, 564, 582, 479, 531, 527, 470, 508, 533, 508, 497, 503, 525, 509, 499, 501, 494, 511, 475, 491, 465, 532, 531, 528, 494, 512, 498, 514, 519, 505, 528, 513, 495, 494, 489, 520, 526, 505, 533, 540, 551, 488, 537] payload_rows_per_dst=[10950, 10968, 10878, 10956, 11010, 10948, 10864, 10982] dup_rows_per_dst=[5410, 5574, 5394, 5456, 5430, 5448, 5272, 5532] combine_rows_per_src=[10950, 10968, 10878, 10956, 11010, 10948, 10864, 10982] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87448 payload_rows_remote=76456 dup_rows=43624 dup_rows_remote=38014 dedup_ratio=0.332825 router_tokens_per_expert=[519, 527, 513, 495, 484, 503, 472, 483, 542, 514, 502, 515, 526, 494, 490, 509, 519, 503, 524, 530, 480, 539, 504, 539, 486, 484, 509, 538, 468, 486, 526, 480, 528, 505, 500, 499, 553, 515, 534, 520, 499, 509, 484, 529, 528, 541, 479, 488, 507, 529, 514, 522, 519, 484, 549, 521, 528, 521, 485, 504, 489, 516, 504, 521, 511, 550, 523, 537, 500, 487, 495, 489, 492, 497, 485, 506, 533, 509, 516, 511, 529, 542, 529, 503, 482, 524, 489, 498, 529, 512, 507, 489, 530, 541, 519, 499, 546, 518, 468, 553, 493, 514, 524, 518, 522, 524, 554, 498, 522, 551, 526, 488, 468, 483, 541, 535, 522, 553, 500, 508, 491, 516, 515, 506, 531, 512, 543, 503] payload_rows_per_dst=[10816, 10890, 10854, 10984, 10896, 11040, 10976, 10992] dup_rows_per_dst=[5360, 5340, 5290, 5396, 5386, 5674, 5568, 5610] combine_rows_per_src=[10816, 10890, 10854, 10984, 10896, 11040, 10976, 10992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87396 payload_rows_remote=76586 dup_rows=43676 dup_rows_remote=38172 dedup_ratio=0.333221 router_tokens_per_expert=[523, 492, 501, 529, 520, 498, 562, 532, 507, 500, 531, 503, 485, 494, 513, 508, 549, 539, 484, 493, 511, 515, 486, 495, 506, 517, 459, 551, 524, 534, 492, 502, 524, 539, 532, 472, 517, 538, 491, 507, 493, 526, 517, 526, 509, 499, 529, 505, 497, 520, 533, 507, 530, 527, 524, 480, 559, 545, 518, 489, 475, 526, 506, 494, 513, 531, 494, 521, 505, 515, 514, 509, 519, 537, 515, 508, 497, 554, 469, 512, 511, 502, 519, 514, 490, 508, 484, 518, 506, 540, 507, 492, 523, 518, 512, 541, 530, 499, 491, 497, 512, 530, 536, 489, 483, 507, 530, 561, 541, 495, 507, 496, 533, 503, 520, 508, 499, 464, 493, 521, 491, 526, 523, 533, 511, 492, 495, 513] payload_rows_per_dst=[10912, 10810, 10926, 10960, 11040, 10882, 10952, 10914] dup_rows_per_dst=[5484, 5504, 5522, 5500, 5386, 5488, 5456, 5336] combine_rows_per_src=[10912, 10810, 10926, 10960, 11040, 10882, 10952, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87524 payload_rows_remote=76602 dup_rows=43548 dup_rows_remote=38034 dedup_ratio=0.332245 router_tokens_per_expert=[495, 495, 478, 483, 537, 501, 530, 525, 509, 485, 517, 486, 521, 516, 533, 540, 485, 515, 498, 518, 453, 515, 494, 515, 512, 485, 502, 487, 501, 506, 520, 537, 543, 471, 513, 521, 517, 522, 544, 493, 508, 492, 529, 491, 555, 481, 521, 528, 549, 509, 503, 507, 521, 506, 525, 491, 488, 544, 512, 501, 529, 519, 501, 499, 510, 532, 490, 507, 486, 507, 510, 533, 486, 509, 517, 497, 528, 484, 521, 507, 509, 487, 545, 511, 509, 497, 518, 511, 511, 528, 530, 545, 519, 494, 511, 493, 529, 528, 502, 537, 549, 537, 541, 536, 495, 523, 514, 514, 536, 547, 509, 536, 546, 527, 552, 491, 522, 542, 499, 519, 529, 485, 497, 470, 521, 455, 514, 465] payload_rows_per_dst=[10934, 10814, 10976, 10902, 10816, 10922, 11156, 11004] dup_rows_per_dst=[5368, 5272, 5482, 5506, 5432, 5514, 5710, 5264] combine_rows_per_src=[10934, 10814, 10976, 10902, 10816, 10922, 11156, 11004] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87692 payload_rows_remote=76828 dup_rows=43380 dup_rows_remote=38126 dedup_ratio=0.330963 router_tokens_per_expert=[503, 502, 521, 537, 505, 489, 562, 515, 462, 537, 490, 574, 499, 516, 497, 530, 517, 478, 532, 504, 486, 491, 527, 492, 508, 488, 503, 527, 518, 493, 488, 490, 540, 514, 512, 568, 513, 497, 506, 533, 543, 511, 496, 496, 505, 461, 540, 507, 565, 521, 516, 492, 512, 506, 506, 517, 488, 484, 503, 491, 540, 495, 538, 526, 487, 492, 517, 496, 535, 510, 509, 519, 488, 489, 529, 496, 490, 489, 496, 517, 530, 505, 505, 546, 556, 550, 541, 539, 478, 539, 494, 508, 488, 505, 483, 558, 544, 494, 521, 540, 497, 500, 515, 476, 499, 547, 529, 496, 501, 524, 525, 518, 479, 490, 562, 489, 524, 502, 547, 512, 504, 526, 523, 492, 505, 538, 520, 490] payload_rows_per_dst=[10962, 10876, 10960, 10970, 10864, 11064, 11010, 10986] dup_rows_per_dst=[5516, 5208, 5524, 5430, 5254, 5586, 5442, 5420] combine_rows_per_src=[10962, 10876, 10960, 10970, 10864, 11064, 11010, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87356 payload_rows_remote=76466 dup_rows=43716 dup_rows_remote=38306 dedup_ratio=0.333527 router_tokens_per_expert=[500, 526, 533, 476, 517, 516, 488, 544, 498, 519, 528, 524, 493, 511, 498, 526, 475, 503, 534, 518, 527, 534, 495, 425, 499, 512, 544, 545, 527, 481, 521, 497, 504, 521, 503, 531, 527, 555, 512, 540, 510, 499, 506, 507, 509, 507, 503, 548, 498, 519, 555, 503, 480, 530, 495, 517, 510, 498, 531, 523, 467, 540, 510, 517, 481, 522, 551, 508, 485, 513, 495, 508, 521, 516, 513, 511, 522, 517, 528, 523, 532, 512, 476, 522, 521, 505, 512, 566, 528, 506, 531, 484, 533, 522, 485, 521, 455, 470, 496, 512, 508, 530, 513, 496, 542, 520, 523, 530, 583, 503, 472, 497, 517, 469, 573, 482, 457, 517, 455, 508, 516, 488, 495, 530, 526, 498, 539, 537] payload_rows_per_dst=[10956, 10816, 10990, 10986, 10962, 10944, 10890, 10812] dup_rows_per_dst=[5438, 5458, 5574, 5400, 5466, 5568, 5410, 5402] combine_rows_per_src=[10956, 10816, 10990, 10986, 10962, 10944, 10890, 10812] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76584 dup_rows=43566 dup_rows_remote=38116 dedup_ratio=0.332382 router_tokens_per_expert=[553, 503, 510, 532, 531, 473, 518, 488, 545, 569, 493, 486, 500, 494, 522, 485, 561, 484, 463, 470, 520, 522, 534, 517, 501, 485, 518, 503, 496, 528, 503, 525, 535, 500, 475, 487, 520, 498, 563, 480, 541, 519, 520, 485, 535, 506, 500, 461, 487, 538, 511, 559, 536, 503, 508, 506, 488, 508, 528, 530, 489, 489, 507, 499, 532, 507, 529, 490, 455, 504, 505, 502, 538, 517, 526, 536, 516, 511, 503, 500, 520, 502, 510, 553, 510, 471, 483, 535, 502, 535, 502, 497, 537, 484, 493, 505, 510, 504, 534, 512, 535, 512, 507, 529, 549, 533, 531, 514, 507, 515, 506, 487, 545, 552, 518, 541, 504, 529, 506, 506, 513, 529, 536, 491, 475, 517, 511, 525] payload_rows_per_dst=[10920, 10908, 10886, 10922, 10950, 11016, 10924, 10980] dup_rows_per_dst=[5484, 5352, 5364, 5450, 5392, 5262, 5646, 5616] combine_rows_per_src=[10920, 10908, 10886, 10922, 10950, 11016, 10924, 10980] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=628146176 dispatch_meta_bytes=459200 combine_payload_bytes=628146176 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628605376 ingress_bytes=779141120 link_bytes=779141120 token_bytes=1256751552 weight_movement_bytes=150994944 total_nvlink_bytes=1407746496 weight_to_token_ratio=0.120147 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.880606126e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626327552 dispatch_meta_bytes=457880 combine_payload_bytes=626327552 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626785432 ingress_bytes=701825024 link_bytes=701825024 token_bytes=1253112984 weight_movement_bytes=75497472 total_nvlink_bytes=1328610456 weight_to_token_ratio=0.060248 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.900129198e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627392512 dispatch_meta_bytes=459032 combine_payload_bytes=627392512 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627851544 ingress_bytes=702889984 link_bytes=702889984 token_bytes=1255244056 weight_movement_bytes=75497472 total_nvlink_bytes=1330741528 weight_to_token_ratio=0.060146 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.913634390e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87342 payload_rows_remote=76360 dup_rows=43730 dup_rows_remote=38150 dedup_ratio=0.333633 router_tokens_per_expert=[526, 522, 498, 518, 510, 544, 502, 519, 503, 516, 527, 477, 504, 495, 513, 538, 516, 460, 502, 519, 536, 582, 520, 495, 490, 548, 491, 500, 506, 525, 515, 525, 503, 514, 519, 513, 498, 541, 499, 497, 518, 532, 529, 461, 529, 539, 520, 514, 489, 557, 468, 520, 502, 496, 504, 508, 486, 490, 530, 509, 491, 524, 515, 538, 562, 507, 509, 520, 536, 515, 485, 524, 503, 545, 502, 491, 503, 536, 482, 531, 489, 525, 495, 484, 535, 568, 503, 525, 536, 500, 510, 474, 554, 513, 505, 553, 512, 513, 460, 524, 500, 542, 505, 487, 530, 532, 526, 496, 467, 561, 475, 480, 539, 523, 516, 507, 485, 537, 536, 493, 514, 470, 492, 473, 506, 512, 499, 509] payload_rows_per_dst=[10982, 11242, 10934, 10880, 11020, 10734, 10712, 10838] dup_rows_per_dst=[5580, 6040, 5518, 5374, 5768, 4982, 5084, 5384] combine_rows_per_src=[10982, 11242, 10934, 10880, 11020, 10734, 10712, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627523584 dispatch_meta_bytes=458544 combine_payload_bytes=627523584 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627982128 ingress_bytes=778518528 link_bytes=778518528 token_bytes=1255505712 weight_movement_bytes=150994944 total_nvlink_bytes=1406500656 weight_to_token_ratio=0.120266 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872710783e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=629374976 dispatch_meta_bytes=459816 combine_payload_bytes=629374976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629834792 ingress_bytes=704872448 link_bytes=704872448 token_bytes=1259209768 weight_movement_bytes=75497472 total_nvlink_bytes=1334707240 weight_to_token_ratio=0.059956 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.938774824e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626409472 dispatch_meta_bytes=459088 combine_payload_bytes=626409472 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626868560 ingress_bytes=928399360 link_bytes=928399360 token_bytes=1253278032 weight_movement_bytes=301989888 total_nvlink_bytes=1555267920 weight_to_token_ratio=0.240960 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177341071e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627376128 dispatch_meta_bytes=458800 combine_payload_bytes=627376128 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627834928 ingress_bytes=778371072 link_bytes=778371072 token_bytes=1255211056 weight_movement_bytes=150994944 total_nvlink_bytes=1406206000 weight_to_token_ratio=0.120294 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.870840833e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625541120 dispatch_meta_bytes=458040 combine_payload_bytes=625541120 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625999160 ingress_bytes=701038592 link_bytes=701038592 token_bytes=1251540280 weight_movement_bytes=75497472 total_nvlink_bytes=1327037752 weight_to_token_ratio=0.060324 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.890156133e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87592 payload_rows_remote=76820 dup_rows=43480 dup_rows_remote=38150 dedup_ratio=0.331726 router_tokens_per_expert=[519, 509, 483, 498, 485, 512, 504, 515, 493, 521, 526, 526, 518, 494, 508, 523, 506, 510, 523, 502, 515, 481, 520, 519, 557, 535, 523, 557, 560, 500, 507, 501, 520, 492, 516, 504, 458, 490, 508, 484, 506, 489, 469, 498, 512, 530, 530, 545, 515, 500, 515, 490, 515, 499, 455, 470, 482, 519, 519, 525, 496, 508, 520, 518, 520, 506, 553, 488, 471, 545, 519, 530, 514, 544, 512, 539, 498, 536, 473, 517, 521, 528, 508, 528, 513, 468, 529, 524, 531, 528, 501, 548, 515, 510, 522, 529, 448, 527, 529, 505, 490, 520, 485, 501, 529, 498, 507, 560, 560, 525, 510, 494, 525, 530, 497, 499, 525, 541, 503, 520, 501, 503, 547, 534, 472, 552, 504, 480] payload_rows_per_dst=[10980, 11112, 10772, 10870, 10990, 10932, 10922, 11014] dup_rows_per_dst=[5288, 5520, 5330, 5222, 5540, 5674, 5454, 5452] combine_rows_per_src=[10980, 11112, 10772, 10870, 10990, 10932, 10922, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87404 payload_rows_remote=76358 dup_rows=43668 dup_rows_remote=38128 dedup_ratio=0.333160 router_tokens_per_expert=[555, 532, 532, 503, 485, 545, 489, 483, 557, 538, 522, 508, 557, 520, 526, 504, 518, 504, 532, 517, 515, 462, 505, 472, 483, 509, 536, 467, 523, 514, 500, 497, 513, 479, 507, 530, 501, 503, 476, 509, 498, 518, 521, 528, 491, 535, 494, 530, 513, 503, 509, 511, 482, 501, 511, 509, 532, 505, 508, 542, 505, 518, 507, 537, 553, 563, 495, 503, 498, 496, 533, 498, 558, 511, 519, 516, 552, 506, 521, 492, 523, 509, 505, 449, 504, 535, 517, 512, 524, 548, 463, 491, 503, 518, 536, 518, 503, 514, 505, 492, 515, 477, 482, 526, 542, 526, 515, 540, 539, 517, 492, 498, 541, 492, 493, 526, 489, 496, 500, 550, 512, 524, 495, 511, 514, 510, 495, 500] payload_rows_per_dst=[11098, 10690, 10916, 10880, 11000, 10850, 10924, 11046] dup_rows_per_dst=[5614, 5128, 5308, 5506, 5628, 5460, 5484, 5540] combine_rows_per_src=[11098, 10690, 10916, 10880, 11000, 10850, 10924, 11046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76598 dup_rows=43566 dup_rows_remote=38142 dedup_ratio=0.332382 router_tokens_per_expert=[457, 518, 529, 548, 501, 540, 581, 505, 473, 517, 510, 516, 506, 518, 544, 526, 528, 446, 528, 519, 536, 509, 519, 485, 500, 516, 515, 519, 544, 477, 490, 535, 510, 495, 487, 512, 504, 482, 507, 538, 503, 515, 559, 540, 539, 481, 494, 525, 486, 530, 527, 508, 501, 527, 527, 502, 508, 488, 531, 490, 507, 525, 526, 467, 509, 523, 528, 502, 509, 530, 547, 538, 507, 501, 516, 494, 518, 493, 505, 488, 475, 521, 512, 525, 506, 513, 509, 516, 493, 562, 522, 470, 519, 522, 507, 500, 499, 491, 543, 528, 509, 480, 506, 499, 510, 505, 482, 535, 501, 474, 504, 506, 495, 544, 540, 534, 522, 499, 531, 516, 553, 501, 503, 525, 526, 515, 509, 475] payload_rows_per_dst=[11006, 10908, 10972, 10994, 11004, 10820, 10848, 10954] dup_rows_per_dst=[5572, 5424, 5410, 5306, 5412, 5524, 5296, 5622] combine_rows_per_src=[11006, 10908, 10972, 10994, 11004, 10820, 10848, 10954] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76464 dup_rows=43590 dup_rows_remote=38022 dedup_ratio=0.332565 router_tokens_per_expert=[499, 499, 508, 522, 541, 531, 480, 489, 513, 488, 495, 535, 538, 530, 529, 504, 526, 505, 509, 497, 489, 487, 522, 536, 501, 529, 478, 538, 531, 475, 512, 515, 538, 472, 566, 523, 524, 521, 499, 514, 516, 494, 488, 500, 537, 543, 526, 514, 545, 499, 501, 522, 497, 491, 469, 487, 521, 483, 505, 479, 527, 507, 511, 539, 495, 490, 516, 554, 509, 502, 550, 551, 484, 454, 529, 510, 479, 543, 470, 502, 531, 485, 492, 511, 520, 530, 531, 499, 489, 527, 511, 537, 541, 537, 561, 491, 547, 521, 490, 541, 510, 522, 519, 501, 525, 497, 480, 542, 515, 504, 501, 501, 519, 527, 517, 554, 501, 492, 520, 496, 502, 523, 506, 477, 483, 548, 517, 498] payload_rows_per_dst=[10946, 10970, 10996, 10836, 10850, 11018, 10966, 10900] dup_rows_per_dst=[5456, 5330, 5554, 5330, 5426, 5568, 5466, 5460] combine_rows_per_src=[10946, 10970, 10996, 10836, 10850, 11018, 10966, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87634 payload_rows_remote=76852 dup_rows=43438 dup_rows_remote=37954 dedup_ratio=0.331406 router_tokens_per_expert=[462, 482, 523, 525, 517, 537, 544, 476, 484, 478, 513, 529, 517, 518, 528, 514, 514, 545, 535, 534, 530, 534, 491, 497, 533, 489, 482, 468, 575, 498, 500, 510, 529, 535, 556, 528, 487, 536, 546, 522, 522, 531, 532, 529, 506, 490, 527, 494, 533, 496, 534, 447, 501, 529, 492, 558, 511, 524, 533, 485, 582, 557, 477, 493, 529, 484, 526, 534, 558, 524, 518, 542, 500, 516, 466, 490, 492, 495, 472, 487, 528, 502, 520, 489, 484, 512, 532, 536, 509, 516, 501, 520, 517, 506, 492, 493, 533, 513, 494, 523, 541, 489, 500, 502, 509, 545, 485, 492, 533, 553, 501, 517, 529, 463, 521, 506, 510, 499, 517, 475, 480, 463, 490, 523, 515, 516, 490, 515] payload_rows_per_dst=[10922, 11024, 11148, 10934, 10782, 10978, 10992, 10854] dup_rows_per_dst=[5372, 5446, 5592, 5570, 5484, 5336, 5468, 5170] combine_rows_per_src=[10922, 11024, 11148, 10934, 10782, 10978, 10992, 10854] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87244 payload_rows_remote=76346 dup_rows=43828 dup_rows_remote=38354 dedup_ratio=0.334381 router_tokens_per_expert=[535, 522, 499, 491, 521, 517, 540, 541, 476, 486, 461, 495, 541, 535, 477, 550, 530, 552, 520, 516, 515, 492, 487, 504, 503, 536, 486, 492, 499, 498, 512, 489, 482, 520, 527, 500, 519, 534, 511, 494, 553, 492, 530, 531, 497, 515, 496, 510, 549, 511, 493, 521, 458, 506, 518, 524, 533, 516, 509, 508, 482, 509, 527, 544, 489, 506, 514, 505, 549, 466, 525, 545, 537, 503, 522, 528, 499, 519, 492, 511, 522, 520, 529, 509, 472, 524, 509, 491, 518, 516, 531, 491, 521, 499, 520, 524, 525, 483, 501, 504, 525, 544, 530, 504, 491, 521, 528, 487, 492, 474, 568, 509, 516, 499, 500, 556, 511, 481, 493, 532, 545, 540, 503, 502, 517, 508, 479, 525] payload_rows_per_dst=[10870, 10792, 10908, 10854, 10968, 10966, 10898, 10988] dup_rows_per_dst=[5504, 5470, 5514, 5562, 5452, 5426, 5474, 5426] combine_rows_per_src=[10870, 10792, 10908, 10854, 10968, 10966, 10898, 10988] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87232 payload_rows_remote=76398 dup_rows=43840 dup_rows_remote=38426 dedup_ratio=0.334473 router_tokens_per_expert=[523, 534, 509, 540, 531, 488, 508, 508, 483, 478, 490, 517, 503, 462, 478, 511, 533, 528, 530, 510, 532, 495, 489, 519, 512, 551, 536, 558, 513, 526, 490, 528, 488, 495, 512, 475, 479, 495, 535, 525, 530, 537, 481, 507, 523, 504, 513, 526, 499, 531, 541, 477, 468, 530, 511, 505, 516, 497, 483, 558, 527, 499, 487, 495, 514, 514, 511, 520, 495, 525, 506, 509, 499, 482, 505, 525, 514, 521, 517, 518, 497, 536, 528, 523, 517, 517, 492, 537, 534, 517, 503, 536, 537, 498, 505, 473, 507, 530, 508, 547, 468, 554, 545, 534, 540, 538, 509, 503, 507, 509, 505, 502, 530, 530, 521, 517, 493, 513, 480, 500, 474, 525, 548, 527, 471, 492, 526, 496] payload_rows_per_dst=[10880, 11078, 10928, 10834, 10828, 10880, 11004, 10800] dup_rows_per_dst=[5246, 5622, 5322, 5414, 5522, 5620, 5608, 5486] combine_rows_per_src=[10880, 11078, 10928, 10834, 10828, 10880, 11004, 10800] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=629309440 dispatch_meta_bytes=459880 combine_payload_bytes=629309440 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=629769320 ingress_bytes=855801856 link_bytes=855801856 token_bytes=1259078760 weight_movement_bytes=226492416 total_nvlink_bytes=1485571176 weight_to_token_ratio=0.179887 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085277217e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625524736 dispatch_meta_bytes=457944 combine_payload_bytes=625524736 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625982680 ingress_bytes=701022208 link_bytes=701022208 token_bytes=1251507416 weight_movement_bytes=75497472 total_nvlink_bytes=1327004888 weight_to_token_ratio=0.060325 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889948361e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87416 payload_rows_remote=76476 dup_rows=43656 dup_rows_remote=38278 dedup_ratio=0.333069 router_tokens_per_expert=[530, 468, 512, 523, 537, 538, 483, 516, 498, 519, 458, 503, 524, 539, 474, 485, 498, 500, 523, 557, 515, 495, 492, 490, 525, 495, 522, 481, 511, 565, 522, 539, 497, 514, 543, 503, 523, 492, 558, 504, 493, 552, 521, 519, 509, 475, 495, 549, 535, 480, 500, 554, 596, 503, 517, 511, 525, 521, 506, 496, 485, 522, 517, 548, 493, 490, 536, 473, 489, 496, 503, 503, 489, 509, 566, 496, 532, 563, 487, 523, 511, 487, 536, 562, 463, 506, 508, 493, 541, 507, 522, 466, 521, 510, 467, 537, 518, 515, 531, 479, 527, 505, 519, 491, 496, 531, 481, 523, 528, 488, 514, 525, 498, 494, 478, 526, 519, 517, 504, 532, 509, 517, 514, 514, 519, 515, 479, 545] payload_rows_per_dst=[10940, 11114, 10908, 11176, 10920, 10674, 10838, 10846] dup_rows_per_dst=[5378, 5444, 5494, 5784, 5266, 5346, 5430, 5514] combine_rows_per_src=[10940, 11114, 10908, 11176, 10920, 10674, 10838, 10846] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627490816 dispatch_meta_bytes=458960 combine_payload_bytes=627490816 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627949776 ingress_bytes=778485760 link_bytes=778485760 token_bytes=1255440592 weight_movement_bytes=150994944 total_nvlink_bytes=1406435536 weight_to_token_ratio=0.120272 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872295238e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626393088 dispatch_meta_bytes=457944 combine_payload_bytes=626393088 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626851032 ingress_bytes=777388032 link_bytes=777388032 token_bytes=1253244120 weight_movement_bytes=150994944 total_nvlink_bytes=1404239064 weight_to_token_ratio=0.120483 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.858374502e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=629571584 dispatch_meta_bytes=459224 combine_payload_bytes=629571584 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=630030808 ingress_bytes=780566528 link_bytes=780566528 token_bytes=1259602392 weight_movement_bytes=150994944 total_nvlink_bytes=1410597336 weight_to_token_ratio=0.119875 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.898682305e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625426432 dispatch_meta_bytes=458800 combine_payload_bytes=625426432 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625885232 ingress_bytes=776421376 link_bytes=776421376 token_bytes=1251311664 weight_movement_bytes=150994944 total_nvlink_bytes=1402306608 weight_to_token_ratio=0.120669 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.846115943e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15053587488 prefetch_bytes=754974720 reduce_bytes=754974720 weight_movement_bytes=1509949440 egress_bytes=7529546784 ingress_bytes=9033990144 link_bytes=9033990144 total_nvlink_bytes=16563536928 weight_to_token_ratio=0.100305 prefetch_share_of_ingress=0.083570 dedup_ratio=0.332798 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=10 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.546975944e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15037097280 prefetch_bytes=830472192 reduce_bytes=377487360 weight_movement_bytes=1207959552 egress_bytes=7521297728 ingress_bytes=8723759104 link_bytes=8723759104 total_nvlink_bytes=16245056832 weight_to_token_ratio=0.080332 prefetch_share_of_ingress=0.095197 dedup_ratio=0.332886 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=5 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.219128755e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625852416 dispatch_meta_bytes=459296 combine_payload_bytes=625852416 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626311712 ingress_bytes=701349888 link_bytes=701349888 token_bytes=1252164128 weight_movement_bytes=75497472 total_nvlink_bytes=1327661600 weight_to_token_ratio=0.060294 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.894103804e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=10 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626491392 dispatch_meta_bytes=459016 combine_payload_bytes=626491392 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626950408 ingress_bytes=701988864 link_bytes=701988864 token_bytes=1253441800 weight_movement_bytes=75497472 total_nvlink_bytes=1328939272 weight_to_token_ratio=0.060232 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.902206920e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15058895584 prefetch_bytes=905969664 reduce_bytes=754974720 weight_movement_bytes=1660944384 egress_bytes=7532200672 ingress_bytes=9187639296 link_bytes=9187639296 total_nvlink_bytes=16719839968 weight_to_token_ratio=0.110297 prefetch_share_of_ingress=0.098607 dedup_ratio=0.332542 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=10 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043045 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.709349904e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15079607440 prefetch_bytes=679477248 reduce_bytes=603979776 weight_movement_bytes=1283457024 egress_bytes=7542557840 ingress_bytes=8820506624 link_bytes=8820964544 total_nvlink_bytes=16363064464 weight_to_token_ratio=0.085112 prefetch_share_of_ingress=0.077034 dedup_ratio=0.331874 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=8 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042739 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.321853905e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15061613064 prefetch_bytes=754974720 reduce_bytes=981467136 weight_movement_bytes=1736441856 egress_bytes=7533558280 ingress_bytes=9264496640 link_bytes=9264496640 total_nvlink_bytes=16798054920 weight_to_token_ratio=0.115289 prefetch_share_of_ingress=0.081491 dedup_ratio=0.332277 prefetch_slots=20 unused_prefetch_slots=364 slot_utilization=0.052083 reduce_slots=13 remote_demand=10 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27178 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.790571513e-05 prefetch_bytes_per_gemm_flop=7.978451797e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15047396296 prefetch_bytes=830472192 reduce_bytes=830472192 weight_movement_bytes=1660944384 egress_bytes=7526452168 ingress_bytes=9181888512 link_bytes=9181888512 total_nvlink_bytes=16708340680 weight_to_token_ratio=0.110381 prefetch_share_of_ingress=0.090447 dedup_ratio=0.333130 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=11 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.703272568e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15048343608 prefetch_bytes=830472192 reduce_bytes=679477248 weight_movement_bytes=1509949440 egress_bytes=7526924344 ingress_bytes=9031368704 link_bytes=9031368704 total_nvlink_bytes=16558293048 weight_to_token_ratio=0.100340 prefetch_share_of_ingress=0.091954 dedup_ratio=0.333003 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=9 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043658 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.544205648e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=10 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15047389248 prefetch_bytes=603979776 reduce_bytes=1207959552 weight_movement_bytes=1811939328 egress_bytes=7526445120 ingress_bytes=9332883456 link_bytes=9332883456 total_nvlink_bytes=16859328576 weight_to_token_ratio=0.120416 prefetch_share_of_ingress=0.064715 dedup_ratio=0.332512 prefetch_slots=16 unused_prefetch_slots=368 slot_utilization=0.041667 reduce_slots=16 remote_demand=8 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043964 gemm_inner_pad_upper_bound=26924 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.862841604e-05 prefetch_bytes_per_gemm_flop=6.382761438e-06 anomalies=none + [2026-08-29 23:36:25] iteration 11/ 12 | consumed samples: 88 | elapsed time per iteration (ms): 354.0 | throughput per GPU (TFLOP/s/GPU): 510.8 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.302853E+00 | loss scale: 1.0 | grad norm: 5.369 | number of skipped iterations: 0 | number of nan iterations: 0 | +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87228 payload_rows_remote=76334 dup_rows=43844 dup_rows_remote=38338 dedup_ratio=0.334503 router_tokens_per_expert=[493, 489, 494, 529, 511, 516, 516, 531, 506, 511, 477, 491, 484, 516, 513, 521, 522, 538, 551, 508, 474, 512, 517, 510, 502, 541, 512, 534, 492, 520, 537, 540, 503, 471, 518, 538, 525, 562, 479, 526, 543, 552, 483, 484, 536, 509, 478, 493, 531, 524, 533, 524, 486, 546, 520, 520, 524, 528, 488, 478, 489, 512, 509, 497, 534, 502, 489, 464, 513, 494, 515, 526, 544, 561, 535, 532, 490, 472, 574, 550, 509, 498, 485, 541, 484, 542, 513, 487, 510, 538, 494, 510, 543, 532, 488, 495, 503, 491, 494, 525, 554, 486, 488, 485, 524, 532, 519, 507, 516, 505, 491, 474, 513, 531, 511, 523, 495, 492, 491, 554, 478, 513, 523, 511, 514, 539, 524, 449] payload_rows_per_dst=[10804, 10894, 10894, 11010, 11044, 10940, 10774, 10868] dup_rows_per_dst=[5392, 5726, 5506, 5408, 5546, 5398, 5414, 5454] combine_rows_per_src=[10804, 10894, 10894, 11010, 11044, 10940, 10774, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87230 payload_rows_remote=76368 dup_rows=43842 dup_rows_remote=38408 dedup_ratio=0.334488 router_tokens_per_expert=[490, 481, 521, 485, 552, 529, 469, 510, 523, 513, 488, 518, 492, 529, 534, 514, 526, 540, 510, 520, 500, 507, 504, 521, 454, 486, 526, 497, 523, 504, 518, 512, 507, 513, 552, 498, 518, 500, 530, 502, 475, 486, 553, 462, 523, 524, 501, 497, 469, 499, 507, 518, 493, 545, 526, 547, 493, 528, 505, 505, 476, 512, 518, 538, 518, 503, 476, 535, 535, 470, 523, 495, 490, 504, 514, 538, 523, 495, 540, 513, 542, 518, 504, 501, 537, 537, 551, 509, 540, 522, 514, 495, 545, 516, 530, 505, 539, 490, 504, 481, 507, 531, 505, 523, 498, 482, 510, 555, 532, 553, 486, 490, 498, 503, 509, 534, 495, 532, 527, 512, 518, 519, 529, 519, 514, 469, 496, 522] payload_rows_per_dst=[10828, 10862, 10900, 10910, 10926, 10996, 10872, 10936] dup_rows_per_dst=[5468, 5434, 5382, 5448, 5418, 5736, 5500, 5456] combine_rows_per_src=[10828, 10862, 10900, 10910, 10926, 10996, 10872, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87492 payload_rows_remote=76622 dup_rows=43580 dup_rows_remote=38176 dedup_ratio=0.332489 router_tokens_per_expert=[545, 496, 526, 520, 506, 489, 501, 558, 529, 515, 522, 520, 491, 532, 511, 491, 540, 494, 465, 559, 476, 515, 521, 561, 516, 508, 499, 557, 519, 566, 490, 548, 501, 532, 523, 531, 524, 521, 516, 503, 522, 507, 517, 475, 542, 525, 510, 522, 519, 468, 506, 502, 529, 518, 433, 516, 502, 530, 473, 518, 506, 481, 496, 495, 483, 522, 529, 529, 486, 539, 524, 513, 500, 549, 528, 515, 519, 526, 502, 552, 541, 513, 522, 530, 529, 504, 504, 518, 468, 486, 550, 497, 493, 489, 476, 486, 501, 518, 487, 502, 506, 475, 524, 527, 505, 514, 509, 500, 541, 518, 500, 501, 495, 504, 527, 519, 524, 527, 482, 525, 510, 482, 515, 492, 521, 518, 463, 533] payload_rows_per_dst=[10720, 10958, 11000, 10842, 11026, 10854, 11222, 10870] dup_rows_per_dst=[5334, 5146, 5542, 5142, 5638, 5358, 6016, 5404] combine_rows_per_src=[10720, 10958, 11000, 10842, 11026, 10854, 11222, 10870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87656 payload_rows_remote=76642 dup_rows=43416 dup_rows_remote=38106 dedup_ratio=0.331238 router_tokens_per_expert=[518, 510, 533, 528, 554, 511, 514, 513, 492, 488, 503, 511, 489, 540, 526, 521, 520, 536, 517, 509, 502, 510, 513, 504, 509, 530, 522, 488, 488, 522, 493, 495, 498, 494, 493, 515, 490, 542, 518, 488, 487, 527, 511, 524, 525, 516, 514, 523, 523, 557, 525, 534, 511, 552, 475, 494, 513, 507, 514, 526, 498, 490, 486, 464, 521, 495, 542, 573, 464, 511, 516, 519, 539, 498, 476, 516, 497, 511, 499, 485, 521, 519, 540, 522, 547, 492, 509, 548, 502, 516, 502, 529, 495, 501, 502, 491, 476, 523, 496, 482, 498, 498, 548, 522, 485, 472, 457, 511, 505, 556, 529, 520, 520, 549, 532, 527, 489, 492, 484, 564, 553, 505, 489, 548, 513, 523, 517, 512] payload_rows_per_dst=[10974, 10912, 10970, 10896, 11014, 11020, 10900, 10970] dup_rows_per_dst=[5528, 5404, 5360, 5442, 5310, 5452, 5256, 5664] combine_rows_per_src=[10974, 10912, 10970, 10896, 11014, 11020, 10900, 10970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87356 payload_rows_remote=76452 dup_rows=43716 dup_rows_remote=38260 dedup_ratio=0.333527 router_tokens_per_expert=[514, 511, 499, 524, 525, 508, 482, 472, 483, 514, 510, 513, 542, 520, 538, 548, 524, 538, 519, 515, 525, 509, 518, 506, 509, 529, 506, 537, 499, 512, 516, 500, 486, 511, 521, 491, 511, 503, 567, 474, 500, 524, 571, 529, 545, 495, 506, 540, 548, 542, 482, 499, 487, 490, 499, 493, 539, 521, 537, 495, 521, 509, 518, 500, 547, 508, 535, 515, 470, 515, 505, 509, 509, 526, 494, 503, 545, 499, 508, 538, 504, 509, 538, 468, 539, 501, 496, 516, 511, 505, 524, 510, 526, 498, 532, 503, 538, 540, 521, 495, 504, 512, 521, 494, 540, 490, 484, 486, 488, 503, 498, 482, 469, 469, 525, 540, 517, 513, 508, 489, 499, 505, 515, 515, 495, 540, 512, 504] payload_rows_per_dst=[10902, 11068, 10944, 11000, 10894, 10904, 10812, 10832] dup_rows_per_dst=[5504, 5456, 5604, 5360, 5558, 5456, 5380, 5398] combine_rows_per_src=[10902, 11068, 10944, 11000, 10894, 10904, 10812, 10832] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87664 payload_rows_remote=76724 dup_rows=43408 dup_rows_remote=38076 dedup_ratio=0.331177 router_tokens_per_expert=[522, 510, 504, 494, 482, 493, 530, 501, 524, 504, 521, 535, 589, 540, 483, 536, 531, 551, 485, 516, 517, 507, 556, 521, 497, 515, 496, 503, 496, 523, 480, 511, 518, 478, 473, 494, 529, 485, 512, 547, 511, 553, 515, 515, 495, 528, 476, 507, 513, 525, 487, 537, 489, 527, 517, 502, 543, 495, 491, 497, 526, 500, 523, 464, 525, 537, 497, 515, 487, 491, 548, 506, 500, 509, 512, 528, 502, 487, 503, 526, 518, 469, 513, 474, 526, 481, 505, 543, 551, 496, 528, 498, 540, 498, 530, 514, 510, 496, 533, 509, 481, 533, 545, 488, 529, 462, 557, 530, 525, 547, 469, 533, 494, 494, 521, 514, 497, 523, 512, 532, 549, 539, 502, 497, 489, 512, 491, 521] payload_rows_per_dst=[11046, 10980, 10884, 10940, 10984, 10934, 11010, 10886] dup_rows_per_dst=[5490, 5430, 5388, 5332, 5362, 5434, 5484, 5488] combine_rows_per_src=[11046, 10980, 10884, 10940, 10984, 10934, 11010, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87476 payload_rows_remote=76554 dup_rows=43596 dup_rows_remote=38164 dedup_ratio=0.332611 router_tokens_per_expert=[450, 530, 488, 510, 508, 518, 521, 517, 516, 536, 491, 504, 517, 530, 519, 520, 500, 552, 498, 531, 529, 525, 493, 547, 516, 494, 530, 482, 525, 537, 505, 525, 516, 475, 523, 529, 521, 483, 530, 502, 503, 466, 496, 504, 483, 507, 515, 502, 510, 497, 509, 484, 554, 521, 517, 502, 523, 501, 500, 467, 505, 524, 523, 497, 530, 522, 517, 541, 550, 511, 533, 477, 499, 501, 528, 541, 533, 525, 522, 535, 538, 489, 507, 507, 538, 509, 528, 507, 504, 501, 521, 497, 509, 516, 518, 509, 533, 490, 524, 533, 479, 480, 510, 521, 509, 511, 492, 503, 516, 492, 503, 534, 525, 505, 468, 506, 549, 535, 485, 502, 510, 504, 513, 502, 527, 521, 508, 530] payload_rows_per_dst=[10922, 11110, 10892, 11130, 10768, 10884, 10914, 10856] dup_rows_per_dst=[5432, 5468, 5382, 5728, 5372, 5422, 5346, 5446] combine_rows_per_src=[10922, 11110, 10892, 11130, 10768, 10884, 10914, 10856] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87504 payload_rows_remote=76638 dup_rows=43568 dup_rows_remote=38262 dedup_ratio=0.332397 router_tokens_per_expert=[523, 546, 511, 516, 504, 534, 544, 511, 497, 482, 532, 557, 528, 516, 539, 524, 506, 482, 473, 541, 488, 510, 517, 512, 530, 520, 499, 523, 508, 493, 513, 497, 507, 510, 478, 490, 512, 506, 502, 522, 558, 524, 513, 490, 535, 541, 500, 524, 495, 538, 490, 517, 526, 520, 524, 498, 504, 514, 519, 518, 550, 503, 502, 524, 514, 479, 498, 499, 538, 520, 463, 491, 513, 509, 481, 517, 496, 522, 529, 537, 500, 519, 482, 516, 523, 455, 524, 552, 497, 511, 489, 515, 547, 495, 504, 513, 554, 510, 503, 509, 513, 476, 518, 506, 505, 519, 532, 495, 502, 489, 482, 473, 564, 501, 527, 483, 562, 524, 501, 528, 502, 492, 553, 504, 489, 523, 494, 525] payload_rows_per_dst=[11088, 10878, 11022, 10982, 10866, 10832, 10866, 10970] dup_rows_per_dst=[5640, 5346, 5402, 5502, 5346, 5452, 5306, 5574] combine_rows_per_src=[11088, 10878, 11022, 10982, 10866, 10832, 10866, 10970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625328128 dispatch_meta_bytes=458688 combine_payload_bytes=625328128 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625786816 ingress_bytes=700825600 link_bytes=700825600 token_bytes=1251114944 weight_movement_bytes=75497472 total_nvlink_bytes=1326612416 weight_to_token_ratio=0.060344 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.887455095e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=625606656 dispatch_meta_bytes=459104 combine_payload_bytes=625606656 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626065760 ingress_bytes=776601600 link_bytes=776601600 token_bytes=1251672416 weight_movement_bytes=150994944 total_nvlink_bytes=1402667360 weight_to_token_ratio=0.120635 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.848401437e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627687424 dispatch_meta_bytes=459192 combine_payload_bytes=627687424 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628146616 ingress_bytes=778682368 link_bytes=778682368 token_bytes=1255834040 weight_movement_bytes=150994944 total_nvlink_bytes=1406828984 weight_to_token_ratio=0.120235 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.874788505e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627851264 dispatch_meta_bytes=458992 combine_payload_bytes=627851264 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628310256 ingress_bytes=778846208 link_bytes=778846208 token_bytes=1256161520 weight_movement_bytes=150994944 total_nvlink_bytes=1407156464 weight_to_token_ratio=0.120203 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.876866226e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=626294784 dispatch_meta_bytes=458848 combine_payload_bytes=626294784 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626753632 ingress_bytes=777289728 link_bytes=777289728 token_bytes=1253048416 weight_movement_bytes=150994944 total_nvlink_bytes=1404043360 weight_to_token_ratio=0.120502 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.857127869e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=628523008 dispatch_meta_bytes=459200 combine_payload_bytes=628523008 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628982208 ingress_bytes=704020480 link_bytes=704020480 token_bytes=1257505216 weight_movement_bytes=75497472 total_nvlink_bytes=1333002688 weight_to_token_ratio=0.060038 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.927970670e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627130368 dispatch_meta_bytes=458872 combine_payload_bytes=627130368 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627589240 ingress_bytes=853622784 link_bytes=853622784 token_bytes=1254719608 weight_movement_bytes=226492416 total_nvlink_bytes=1481212024 weight_to_token_ratio=0.180512 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082513847e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=1 plans=1 dispatch_payload_bytes=627818496 dispatch_meta_bytes=459600 combine_payload_bytes=627818496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628278096 ingress_bytes=703315968 link_bytes=703315968 token_bytes=1256096592 weight_movement_bytes=75497472 total_nvlink_bytes=1331594064 weight_to_token_ratio=0.060105 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.919036466e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87424 payload_rows_remote=76544 dup_rows=43648 dup_rows_remote=38272 dedup_ratio=0.333008 router_tokens_per_expert=[539, 516, 527, 547, 541, 475, 552, 488, 496, 539, 530, 468, 479, 537, 505, 471, 570, 523, 488, 531, 509, 517, 512, 520, 513, 537, 544, 504, 539, 499, 542, 521, 521, 543, 480, 492, 497, 509, 502, 517, 540, 491, 507, 502, 515, 528, 494, 490, 525, 526, 504, 481, 532, 487, 482, 530, 500, 488, 514, 530, 490, 513, 531, 484, 512, 527, 505, 531, 495, 518, 502, 487, 511, 524, 506, 522, 503, 551, 539, 489, 509, 500, 542, 495, 516, 540, 496, 538, 512, 525, 494, 504, 492, 524, 492, 480, 471, 546, 486, 517, 537, 477, 496, 526, 505, 475, 512, 528, 528, 491, 498, 524, 505, 517, 524, 495, 508, 517, 522, 501, 509, 507, 513, 513, 532, 522, 493, 536] payload_rows_per_dst=[10882, 11034, 10880, 10830, 11018, 10922, 10894, 10964] dup_rows_per_dst=[5538, 5704, 5376, 5404, 5426, 5396, 5340, 5464] combine_rows_per_src=[10882, 11034, 10880, 10830, 11018, 10922, 10894, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87620 payload_rows_remote=76754 dup_rows=43452 dup_rows_remote=38102 dedup_ratio=0.331512 router_tokens_per_expert=[502, 533, 546, 471, 509, 554, 494, 495, 559, 512, 493, 505, 539, 491, 506, 496, 503, 528, 486, 480, 496, 513, 504, 523, 509, 509, 489, 505, 509, 514, 519, 521, 549, 495, 531, 532, 502, 491, 490, 535, 513, 486, 514, 479, 508, 506, 546, 551, 528, 537, 524, 496, 570, 521, 497, 497, 523, 533, 505, 539, 529, 530, 526, 506, 516, 489, 506, 515, 529, 486, 535, 532, 494, 480, 470, 473, 561, 478, 510, 504, 542, 516, 499, 516, 523, 476, 521, 532, 528, 509, 569, 514, 534, 501, 504, 509, 534, 490, 485, 512, 515, 519, 506, 497, 502, 463, 498, 491, 498, 530, 496, 530, 505, 511, 522, 513, 498, 525, 527, 498, 518, 497, 490, 530, 518, 501, 516, 528] payload_rows_per_dst=[10936, 10866, 10952, 11108, 10942, 10988, 10914, 10914] dup_rows_per_dst=[5474, 5350, 5504, 5614, 5214, 5598, 5218, 5480] combine_rows_per_src=[10936, 10866, 10952, 11108, 10942, 10988, 10914, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87290 payload_rows_remote=76292 dup_rows=43782 dup_rows_remote=38284 dedup_ratio=0.334030 router_tokens_per_expert=[497, 505, 501, 481, 552, 512, 511, 515, 551, 519, 525, 510, 528, 525, 485, 548, 536, 523, 516, 511, 506, 534, 528, 573, 466, 536, 543, 503, 565, 509, 473, 503, 494, 492, 475, 503, 548, 482, 515, 503, 526, 502, 508, 504, 440, 493, 502, 491, 516, 527, 502, 499, 485, 541, 547, 508, 508, 494, 522, 537, 551, 485, 498, 499, 504, 514, 532, 471, 498, 505, 537, 498, 560, 502, 517, 523, 517, 494, 528, 510, 522, 508, 524, 505, 513, 547, 522, 493, 487, 521, 506, 481, 514, 487, 473, 519, 521, 517, 520, 498, 502, 538, 514, 521, 471, 519, 519, 516, 504, 503, 513, 493, 541, 521, 512, 519, 504, 546, 508, 521, 493, 504, 477, 552, 518, 507, 493, 532] payload_rows_per_dst=[10804, 10666, 10848, 11034, 10998, 10854, 11088, 10998] dup_rows_per_dst=[5394, 5308, 5440, 5404, 5648, 5390, 5700, 5498] combine_rows_per_src=[10804, 10666, 10848, 11034, 10998, 10854, 11088, 10998] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87298 payload_rows_remote=76408 dup_rows=43774 dup_rows_remote=38396 dedup_ratio=0.333969 router_tokens_per_expert=[518, 508, 468, 503, 523, 498, 546, 518, 498, 510, 540, 532, 494, 532, 493, 539, 500, 485, 478, 509, 525, 501, 527, 492, 530, 505, 526, 536, 496, 502, 505, 492, 486, 519, 494, 499, 535, 505, 500, 516, 495, 509, 505, 488, 544, 503, 562, 508, 516, 519, 538, 504, 552, 505, 483, 540, 509, 553, 524, 509, 505, 564, 533, 506, 505, 529, 515, 491, 508, 488, 516, 519, 491, 540, 483, 507, 493, 505, 539, 502, 510, 509, 516, 516, 487, 480, 511, 497, 494, 502, 529, 569, 528, 486, 502, 498, 545, 533, 513, 499, 491, 506, 521, 554, 539, 477, 501, 549, 494, 487, 535, 497, 498, 496, 541, 496, 487, 527, 499, 550, 492, 530, 502, 487, 533, 513, 541, 481] payload_rows_per_dst=[10916, 10862, 10778, 11018, 10956, 10890, 11030, 10848] dup_rows_per_dst=[5524, 5356, 5558, 5702, 5306, 5378, 5452, 5498] combine_rows_per_src=[10916, 10862, 10778, 11018, 10956, 10890, 11030, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87426 payload_rows_remote=76408 dup_rows=43646 dup_rows_remote=38252 dedup_ratio=0.332993 router_tokens_per_expert=[524, 493, 506, 489, 525, 529, 474, 513, 507, 532, 544, 511, 530, 493, 531, 513, 497, 517, 535, 522, 568, 482, 526, 551, 515, 512, 539, 501, 476, 525, 530, 530, 534, 503, 559, 505, 518, 501, 517, 538, 456, 513, 553, 526, 523, 529, 522, 522, 483, 522, 529, 503, 521, 508, 529, 504, 490, 489, 495, 537, 519, 481, 505, 456, 516, 494, 522, 517, 498, 519, 532, 500, 496, 521, 519, 512, 505, 530, 510, 515, 495, 515, 519, 500, 503, 455, 510, 487, 500, 538, 521, 489, 533, 483, 508, 500, 482, 479, 566, 476, 502, 508, 552, 529, 495, 494, 509, 528, 502, 520, 503, 536, 508, 489, 550, 515, 489, 484, 525, 520, 501, 557, 504, 507, 496, 528, 521, 469] payload_rows_per_dst=[10838, 11034, 11004, 10782, 11018, 10820, 10964, 10966] dup_rows_per_dst=[5590, 5618, 5634, 5360, 5394, 5292, 5398, 5360] combine_rows_per_src=[10838, 11034, 11004, 10782, 11018, 10820, 10964, 10966] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87388 payload_rows_remote=76532 dup_rows=43684 dup_rows_remote=38242 dedup_ratio=0.333282 router_tokens_per_expert=[519, 553, 480, 527, 485, 482, 521, 470, 515, 496, 466, 521, 504, 475, 544, 527, 513, 533, 512, 509, 523, 491, 546, 516, 551, 492, 532, 506, 504, 482, 472, 483, 468, 505, 530, 478, 552, 480, 529, 512, 548, 512, 478, 530, 499, 487, 514, 525, 509, 500, 512, 486, 538, 499, 519, 521, 517, 468, 514, 503, 512, 531, 509, 511, 504, 517, 500, 523, 480, 523, 510, 526, 502, 519, 530, 547, 513, 525, 506, 521, 474, 538, 530, 516, 466, 490, 510, 543, 535, 519, 528, 511, 505, 498, 539, 468, 494, 537, 516, 534, 505, 527, 523, 507, 487, 521, 498, 532, 530, 549, 537, 510, 533, 537, 497, 529, 498, 454, 500, 536, 493, 523, 487, 505, 534, 535, 565, 541] payload_rows_per_dst=[10982, 10840, 10934, 10856, 10908, 10892, 11042, 10934] dup_rows_per_dst=[5188, 5490, 5360, 5442, 5584, 5448, 5572, 5600] combine_rows_per_src=[10982, 10840, 10934, 10856, 10908, 10892, 11042, 10934] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=627048448 dispatch_meta_bytes=459264 combine_payload_bytes=627048448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627507712 ingress_bytes=702545920 link_bytes=702545920 token_bytes=1254556160 weight_movement_bytes=75497472 total_nvlink_bytes=1330053632 weight_to_token_ratio=0.060179 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909271174e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87574 payload_rows_remote=76662 dup_rows=43498 dup_rows_remote=38076 dedup_ratio=0.331863 router_tokens_per_expert=[530, 514, 525, 541, 516, 489, 499, 506, 493, 484, 537, 512, 559, 502, 539, 506, 500, 523, 538, 530, 520, 513, 530, 501, 525, 486, 515, 471, 516, 511, 536, 522, 496, 512, 520, 484, 491, 517, 483, 519, 590, 528, 496, 485, 516, 459, 515, 530, 529, 550, 491, 498, 522, 530, 445, 515, 507, 490, 494, 509, 510, 490, 518, 507, 531, 508, 520, 509, 532, 531, 500, 505, 504, 483, 519, 516, 523, 490, 482, 544, 460, 561, 526, 531, 491, 532, 538, 540, 503, 529, 512, 532, 509, 555, 477, 525, 505, 523, 520, 481, 478, 503, 549, 504, 541, 514, 508, 519, 550, 502, 472, 498, 537, 503, 455, 534, 543, 536, 489, 498, 491, 523, 494, 504, 490, 528, 517, 474] payload_rows_per_dst=[10918, 11028, 10840, 10862, 10954, 11184, 10912, 10876] dup_rows_per_dst=[5586, 5446, 5442, 5348, 5440, 5458, 5422, 5356] combine_rows_per_src=[10918, 11028, 10840, 10862, 10954, 11184, 10912, 10876] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628768768 dispatch_meta_bytes=459424 combine_payload_bytes=628768768 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=629228192 ingress_bytes=930758656 link_bytes=930758656 token_bytes=1257996960 weight_movement_bytes=301989888 total_nvlink_bytes=1559986848 weight_to_token_ratio=0.240056 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.180332990e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=624984064 dispatch_meta_bytes=458304 combine_payload_bytes=624984064 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=625442368 ingress_bytes=851476480 link_bytes=851476480 token_bytes=1250426432 weight_movement_bytes=226492416 total_nvlink_bytes=1476918848 weight_to_token_ratio=0.181132 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.079792031e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=459216 combine_payload_bytes=625934336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626393552 ingress_bytes=701431808 link_bytes=701431808 token_bytes=1252327888 weight_movement_bytes=75497472 total_nvlink_bytes=1327825360 weight_to_token_ratio=0.060286 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895142665e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87378 payload_rows_remote=76408 dup_rows=43694 dup_rows_remote=38162 dedup_ratio=0.333359 router_tokens_per_expert=[490, 528, 530, 544, 518, 517, 494, 520, 470, 541, 551, 516, 495, 521, 516, 472, 486, 529, 508, 499, 487, 505, 513, 543, 518, 530, 519, 511, 535, 534, 519, 519, 550, 515, 522, 514, 496, 539, 537, 517, 475, 511, 505, 526, 537, 500, 523, 494, 534, 487, 575, 506, 486, 496, 497, 508, 503, 449, 502, 512, 544, 512, 511, 497, 461, 539, 488, 475, 504, 520, 517, 549, 505, 527, 522, 501, 481, 519, 507, 518, 487, 503, 488, 520, 499, 528, 535, 541, 529, 509, 517, 540, 506, 495, 491, 539, 479, 502, 494, 476, 516, 484, 479, 510, 503, 484, 561, 535, 515, 505, 488, 532, 503, 497, 526, 461, 510, 517, 518, 560, 559, 549, 547, 526, 503, 502, 476, 501] payload_rows_per_dst=[10970, 10892, 10964, 10860, 10952, 11008, 10840, 10892] dup_rows_per_dst=[5532, 5578, 5558, 5448, 5314, 5554, 5286, 5424] combine_rows_per_src=[10970, 10892, 10964, 10860, 10952, 11008, 10840, 10892] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=458640 combine_payload_bytes=625934336 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626392976 ingress_bytes=701431808 link_bytes=701431808 token_bytes=1252327312 weight_movement_bytes=75497472 total_nvlink_bytes=1327824784 weight_to_token_ratio=0.060286 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.895142665e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=626950144 dispatch_meta_bytes=459096 combine_payload_bytes=626950144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627409240 ingress_bytes=702447616 link_bytes=702447616 token_bytes=1254359384 weight_movement_bytes=75497472 total_nvlink_bytes=1329856856 weight_to_token_ratio=0.060188 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908024541e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=628015104 dispatch_meta_bytes=458952 combine_payload_bytes=628015104 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628474056 ingress_bytes=703512576 link_bytes=703512576 token_bytes=1256489160 weight_movement_bytes=75497472 total_nvlink_bytes=1331986632 weight_to_token_ratio=0.060086 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.921529732e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87326 payload_rows_remote=76394 dup_rows=43746 dup_rows_remote=38268 dedup_ratio=0.333755 router_tokens_per_expert=[505, 541, 504, 525, 534, 537, 548, 532, 560, 520, 524, 501, 537, 525, 474, 502, 526, 477, 487, 489, 526, 512, 518, 521, 466, 517, 516, 537, 513, 529, 516, 478, 519, 498, 486, 523, 559, 514, 492, 490, 502, 531, 498, 534, 506, 498, 529, 526, 493, 536, 504, 519, 497, 495, 492, 526, 479, 495, 496, 549, 490, 528, 503, 518, 504, 507, 507, 502, 534, 543, 493, 511, 520, 496, 504, 534, 462, 483, 494, 492, 487, 505, 491, 472, 540, 510, 536, 506, 507, 493, 524, 507, 520, 512, 531, 512, 539, 540, 528, 495, 498, 477, 555, 498, 471, 530, 510, 518, 515, 547, 504, 552, 535, 509, 519, 516, 480, 464, 507, 518, 531, 514, 527, 520, 537, 514, 517, 490] payload_rows_per_dst=[11080, 10822, 10932, 10896, 10874, 10906, 10914, 10902] dup_rows_per_dst=[5658, 5434, 5478, 5344, 5298, 5400, 5640, 5494] combine_rows_per_src=[11080, 10822, 10932, 10896, 10874, 10906, 10914, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=2 plans=1 dispatch_payload_bytes=625934336 dispatch_meta_bytes=458280 combine_payload_bytes=625934336 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626392616 ingress_bytes=776929280 link_bytes=776929280 token_bytes=1252326952 weight_movement_bytes=150994944 total_nvlink_bytes=1403321896 weight_to_token_ratio=0.120572 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.852556881e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87294 payload_rows_remote=76356 dup_rows=43778 dup_rows_remote=38344 dedup_ratio=0.334000 router_tokens_per_expert=[502, 511, 511, 518, 476, 513, 497, 520, 537, 540, 519, 526, 534, 498, 512, 518, 525, 468, 521, 519, 509, 508, 489, 493, 562, 504, 520, 462, 517, 520, 561, 508, 491, 558, 505, 477, 529, 524, 534, 497, 491, 527, 551, 516, 486, 503, 530, 526, 525, 502, 494, 554, 472, 542, 557, 511, 544, 534, 453, 487, 457, 482, 536, 481, 489, 522, 503, 509, 544, 480, 533, 532, 536, 465, 548, 507, 574, 482, 517, 508, 551, 510, 515, 498, 510, 521, 560, 514, 494, 488, 503, 544, 483, 502, 501, 506, 477, 521, 531, 509, 516, 499, 487, 488, 521, 492, 525, 478, 526, 524, 521, 462, 508, 526, 510, 524, 518, 504, 509, 562, 519, 519, 537, 457, 489, 491, 522, 521] payload_rows_per_dst=[10858, 10938, 10970, 10766, 10962, 10900, 10864, 11036] dup_rows_per_dst=[5606, 5434, 5520, 5496, 5536, 5500, 5290, 5396] combine_rows_per_src=[10858, 10938, 10970, 10766, 10962, 10900, 10864, 11036] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87280 payload_rows_remote=76396 dup_rows=43792 dup_rows_remote=38254 dedup_ratio=0.334106 router_tokens_per_expert=[478, 510, 485, 553, 561, 492, 515, 550, 544, 487, 507, 503, 489, 490, 494, 508, 561, 526, 496, 492, 543, 523, 530, 542, 526, 477, 515, 496, 495, 478, 466, 556, 544, 516, 539, 500, 516, 485, 525, 519, 510, 507, 517, 514, 493, 558, 512, 557, 498, 528, 461, 535, 499, 487, 500, 505, 532, 496, 517, 516, 501, 540, 500, 502, 495, 515, 509, 518, 537, 501, 496, 566, 451, 546, 528, 500, 467, 490, 518, 469, 504, 493, 540, 521, 515, 500, 540, 491, 522, 496, 513, 568, 539, 542, 544, 512, 487, 520, 490, 559, 475, 505, 516, 510, 482, 511, 507, 511, 500, 523, 519, 464, 557, 512, 527, 484, 540, 534, 517, 488, 483, 483, 497, 543, 539, 483, 536, 471] payload_rows_per_dst=[10922, 10818, 10930, 10922, 10846, 11088, 10870, 10884] dup_rows_per_dst=[5368, 5626, 5430, 5380, 5366, 5796, 5288, 5538] combine_rows_per_src=[10922, 10818, 10930, 10922, 10846, 11088, 10870, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625819648 dispatch_meta_bytes=458648 combine_payload_bytes=625819648 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626278296 ingress_bytes=852312064 link_bytes=852312064 token_bytes=1252097944 weight_movement_bytes=226492416 total_nvlink_bytes=1478590360 weight_to_token_ratio=0.180890 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080851669e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87664 payload_rows_remote=76714 dup_rows=43408 dup_rows_remote=37976 dedup_ratio=0.331177 router_tokens_per_expert=[532, 492, 500, 516, 536, 463, 548, 531, 500, 510, 465, 508, 551, 523, 523, 490, 514, 506, 494, 544, 516, 478, 477, 487, 508, 512, 517, 517, 491, 507, 515, 491, 484, 499, 501, 503, 492, 506, 537, 503, 506, 519, 490, 530, 503, 528, 497, 508, 553, 468, 519, 513, 481, 504, 508, 511, 497, 503, 559, 515, 518, 503, 512, 553, 537, 473, 528, 506, 515, 499, 543, 521, 537, 551, 531, 525, 527, 515, 496, 513, 511, 518, 539, 501, 513, 523, 507, 507, 475, 521, 523, 492, 507, 491, 546, 517, 504, 518, 498, 535, 520, 484, 527, 490, 526, 528, 511, 518, 503, 508, 529, 535, 477, 519, 543, 549, 516, 484, 511, 512, 498, 527, 528, 535, 479, 502, 530, 499] payload_rows_per_dst=[10966, 10842, 10920, 11010, 11106, 10950, 10928, 10942] dup_rows_per_dst=[5410, 5306, 5292, 5424, 5528, 5432, 5540, 5476] combine_rows_per_src=[10966, 10842, 10920, 11010, 11106, 10950, 10928, 10942] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 1, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87466 payload_rows_remote=76636 dup_rows=43606 dup_rows_remote=38036 dedup_ratio=0.332687 router_tokens_per_expert=[516, 523, 530, 480, 495, 523, 536, 506, 508, 521, 530, 491, 569, 525, 497, 503, 527, 482, 486, 479, 527, 514, 534, 504, 521, 497, 502, 516, 520, 504, 541, 519, 497, 500, 525, 503, 491, 476, 505, 543, 505, 466, 550, 519, 503, 511, 528, 522, 487, 527, 513, 477, 497, 514, 544, 525, 473, 499, 516, 517, 529, 519, 483, 476, 509, 507, 528, 487, 508, 519, 512, 528, 537, 528, 478, 502, 487, 535, 509, 526, 489, 528, 491, 520, 518, 514, 528, 535, 489, 511, 528, 495, 550, 551, 482, 533, 516, 488, 501, 550, 508, 481, 533, 484, 526, 495, 493, 536, 495, 491, 506, 528, 514, 501, 500, 518, 501, 518, 536, 546, 491, 537, 525, 527, 526, 516, 515, 506] payload_rows_per_dst=[11080, 10942, 10862, 10810, 10830, 11038, 10864, 11040] dup_rows_per_dst=[5426, 5404, 5426, 5382, 5570, 5486, 5398, 5514] combine_rows_per_src=[11080, 10942, 10862, 10810, 10830, 11038, 10864, 11040] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625508352 dispatch_meta_bytes=458800 combine_payload_bytes=625508352 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625967152 ingress_bytes=701005824 link_bytes=701005824 token_bytes=1251475504 weight_movement_bytes=75497472 total_nvlink_bytes=1326972976 weight_to_token_ratio=0.060327 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889740589e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87408 payload_rows_remote=76436 dup_rows=43664 dup_rows_remote=38196 dedup_ratio=0.333130 router_tokens_per_expert=[510, 534, 464, 518, 541, 484, 551, 516, 517, 501, 516, 521, 470, 536, 530, 479, 516, 504, 524, 491, 494, 543, 485, 489, 512, 528, 510, 485, 526, 526, 509, 491, 514, 494, 491, 514, 516, 499, 515, 531, 505, 548, 500, 476, 562, 520, 501, 505, 522, 543, 512, 476, 507, 504, 538, 495, 480, 505, 518, 552, 530, 541, 486, 511, 496, 479, 496, 537, 534, 526, 511, 494, 506, 521, 521, 507, 560, 480, 515, 460, 505, 509, 475, 501, 495, 541, 508, 510, 492, 498, 502, 503, 486, 545, 514, 466, 516, 518, 517, 471, 551, 504, 547, 538, 511, 507, 524, 559, 545, 493, 544, 489, 501, 520, 504, 499, 522, 528, 524, 545, 513, 530, 485, 549, 497, 528, 508, 524] payload_rows_per_dst=[10844, 10842, 10922, 10972, 10898, 10872, 11090, 10968] dup_rows_per_dst=[5532, 5424, 5460, 5468, 5388, 5228, 5578, 5586] combine_rows_per_src=[10844, 10842, 10922, 10972, 10898, 10872, 11090, 10968] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87210 payload_rows_remote=76334 dup_rows=43862 dup_rows_remote=38392 dedup_ratio=0.334641 router_tokens_per_expert=[506, 478, 497, 517, 530, 501, 511, 452, 478, 502, 496, 514, 515, 493, 498, 526, 514, 499, 517, 535, 524, 559, 555, 477, 494, 512, 514, 511, 567, 482, 497, 513, 520, 456, 511, 488, 533, 513, 519, 547, 547, 521, 484, 525, 501, 537, 537, 502, 509, 534, 493, 496, 509, 557, 529, 492, 515, 510, 516, 479, 549, 500, 493, 514, 507, 477, 507, 501, 534, 519, 516, 483, 520, 525, 533, 549, 492, 509, 485, 480, 476, 532, 526, 515, 511, 498, 472, 551, 492, 516, 479, 528, 506, 486, 542, 492, 522, 510, 467, 516, 554, 551, 495, 557, 525, 500, 499, 489, 491, 499, 497, 501, 532, 511, 507, 529, 548, 526, 542, 564, 533, 508, 493, 511, 512, 498, 518, 552] payload_rows_per_dst=[10782, 10976, 10926, 10896, 10874, 10866, 10876, 11014] dup_rows_per_dst=[5246, 5564, 5556, 5494, 5400, 5378, 5470, 5754] combine_rows_per_src=[10782, 10976, 10926, 10896, 10874, 10866, 10876, 11014] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=458600 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626294632 ingress_bytes=927825920 link_bytes=927825920 token_bytes=1252130664 weight_movement_bytes=301989888 total_nvlink_bytes=1554120552 weight_to_token_ratio=0.241181 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.176613868e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=628441088 dispatch_meta_bytes=458760 combine_payload_bytes=628441088 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628899848 ingress_bytes=703938560 link_bytes=703938560 token_bytes=1257340936 weight_movement_bytes=75497472 total_nvlink_bytes=1332838408 weight_to_token_ratio=0.060045 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.926931809e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626163712 dispatch_meta_bytes=458528 combine_payload_bytes=626163712 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626622240 ingress_bytes=701661184 link_bytes=701661184 token_bytes=1252785952 weight_movement_bytes=75497472 total_nvlink_bytes=1328283424 weight_to_token_ratio=0.060264 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898051476e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=458688 combine_payload_bytes=627802112 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628260800 ingress_bytes=703299584 link_bytes=703299584 token_bytes=1256062912 weight_movement_bytes=75497472 total_nvlink_bytes=1331560384 weight_to_token_ratio=0.060106 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918828694e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=625328128 dispatch_meta_bytes=458904 combine_payload_bytes=625328128 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625787032 ingress_bytes=700825600 link_bytes=700825600 token_bytes=1251115160 weight_movement_bytes=75497472 total_nvlink_bytes=1326612632 weight_to_token_ratio=0.060344 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.887455095e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87366 payload_rows_remote=76490 dup_rows=43706 dup_rows_remote=38288 dedup_ratio=0.333450 router_tokens_per_expert=[526, 490, 512, 564, 545, 476, 506, 472, 542, 490, 492, 487, 504, 491, 504, 546, 530, 491, 494, 465, 498, 487, 519, 517, 508, 504, 511, 524, 506, 497, 527, 531, 491, 557, 515, 534, 498, 516, 523, 472, 493, 535, 512, 499, 531, 521, 534, 493, 546, 505, 511, 541, 516, 578, 546, 528, 532, 515, 530, 513, 500, 523, 537, 485, 526, 525, 509, 524, 527, 504, 483, 524, 523, 521, 512, 524, 513, 493, 502, 520, 479, 515, 475, 535, 494, 505, 491, 505, 521, 529, 518, 490, 492, 504, 534, 529, 497, 509, 497, 510, 532, 486, 555, 511, 515, 533, 488, 520, 497, 473, 508, 526, 518, 525, 507, 544, 488, 510, 482, 484, 533, 519, 502, 468, 481, 514, 547, 525] payload_rows_per_dst=[10876, 11012, 10948, 11036, 11058, 10854, 10972, 10610] dup_rows_per_dst=[5418, 5688, 5500, 5776, 5538, 5378, 5490, 4918] combine_rows_per_src=[10876, 11012, 10948, 11036, 11058, 10854, 10972, 10610] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87518 payload_rows_remote=76600 dup_rows=43554 dup_rows_remote=38190 dedup_ratio=0.332291 router_tokens_per_expert=[522, 504, 539, 540, 539, 489, 474, 501, 519, 523, 520, 503, 545, 502, 518, 514, 539, 478, 542, 515, 532, 497, 521, 495, 514, 512, 523, 558, 503, 550, 495, 507, 527, 481, 501, 532, 514, 514, 500, 501, 495, 502, 524, 477, 511, 520, 508, 534, 520, 494, 518, 501, 531, 520, 514, 482, 515, 500, 528, 497, 457, 530, 486, 527, 492, 480, 516, 484, 490, 519, 555, 508, 515, 523, 546, 538, 515, 509, 504, 536, 476, 518, 538, 510, 512, 546, 492, 521, 520, 504, 527, 529, 476, 476, 530, 525, 490, 520, 502, 521, 500, 515, 514, 509, 487, 496, 558, 501, 479, 485, 523, 492, 503, 519, 528, 505, 527, 525, 501, 525, 528, 494, 537, 498, 530, 533, 466, 501] payload_rows_per_dst=[11036, 11014, 10918, 10852, 10886, 10948, 10836, 11028] dup_rows_per_dst=[5468, 5548, 5364, 5388, 5574, 5452, 5348, 5412] combine_rows_per_src=[11036, 11014, 10918, 10852, 10886, 10948, 10836, 11028] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87510 payload_rows_remote=76530 dup_rows=43562 dup_rows_remote=38212 dedup_ratio=0.332352 router_tokens_per_expert=[527, 508, 563, 496, 464, 482, 528, 523, 538, 530, 516, 529, 489, 480, 480, 555, 497, 493, 506, 541, 501, 503, 484, 525, 560, 504, 548, 507, 528, 488, 503, 477, 530, 517, 510, 509, 505, 519, 523, 523, 492, 511, 490, 534, 519, 480, 511, 487, 517, 504, 478, 489, 538, 514, 537, 543, 509, 511, 486, 526, 500, 486, 516, 507, 494, 497, 533, 562, 495, 507, 522, 524, 524, 516, 478, 510, 498, 489, 536, 513, 497, 479, 577, 542, 456, 518, 449, 522, 519, 538, 485, 533, 552, 522, 549, 512, 492, 533, 565, 507, 502, 484, 533, 526, 491, 518, 555, 532, 509, 493, 530, 519, 513, 502, 529, 480, 514, 504, 508, 457, 511, 508, 489, 520, 475, 534, 540, 521] payload_rows_per_dst=[10940, 10980, 10910, 10936, 10892, 10986, 10986, 10880] dup_rows_per_dst=[5476, 5350, 5410, 5386, 5504, 5514, 5592, 5330] combine_rows_per_src=[10940, 10980, 10910, 10936, 10892, 10986, 10986, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=3 plans=1 dispatch_payload_bytes=626606080 dispatch_meta_bytes=459112 combine_payload_bytes=626606080 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=627065192 ingress_bytes=777601024 link_bytes=777601024 token_bytes=1253671272 weight_movement_bytes=150994944 total_nvlink_bytes=1404666216 weight_to_token_ratio=0.120442 gemm_rows_defined=66176 gemm_rows_charged=69632 gemm_tail_pad_rows=3456 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.049632 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861075541e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87476 payload_rows_remote=76540 dup_rows=43596 dup_rows_remote=37946 dedup_ratio=0.332611 router_tokens_per_expert=[484, 507, 510, 507, 483, 508, 537, 528, 538, 530, 501, 490, 476, 546, 548, 491, 499, 496, 493, 570, 491, 507, 486, 520, 502, 497, 505, 530, 521, 511, 516, 480, 528, 521, 501, 549, 500, 519, 512, 514, 540, 547, 520, 526, 511, 515, 513, 472, 525, 508, 508, 520, 492, 532, 505, 519, 519, 524, 488, 490, 506, 530, 508, 522, 514, 530, 550, 488, 527, 527, 492, 533, 490, 482, 536, 511, 523, 496, 482, 501, 490, 488, 529, 531, 530, 544, 518, 554, 525, 506, 478, 554, 526, 483, 536, 488, 510, 532, 505, 505, 492, 489, 516, 538, 525, 488, 534, 540, 521, 498, 503, 491, 504, 516, 517, 490, 494, 493, 534, 495, 502, 527, 535, 478, 499, 505, 501, 505] payload_rows_per_dst=[10596, 10860, 11008, 10956, 11118, 11002, 11000, 10936] dup_rows_per_dst=[5072, 5390, 5532, 5436, 5540, 5558, 5418, 5650] combine_rows_per_src=[10596, 10860, 11008, 10956, 11118, 11002, 11000, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627507200 dispatch_meta_bytes=459160 combine_payload_bytes=627507200 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627966360 ingress_bytes=778502144 link_bytes=778502144 token_bytes=1255473560 weight_movement_bytes=150994944 total_nvlink_bytes=1406468504 weight_to_token_ratio=0.120269 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.872503011e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87510 payload_rows_remote=76592 dup_rows=43562 dup_rows_remote=38118 dedup_ratio=0.332352 router_tokens_per_expert=[536, 508, 509, 499, 518, 526, 479, 522, 523, 506, 536, 539, 460, 519, 506, 526, 539, 500, 500, 533, 487, 531, 511, 482, 515, 517, 505, 515, 486, 504, 538, 520, 500, 482, 486, 487, 524, 600, 499, 500, 468, 483, 545, 465, 500, 472, 513, 539, 489, 551, 552, 495, 533, 503, 506, 507, 484, 514, 499, 500, 525, 541, 506, 540, 489, 476, 499, 527, 524, 520, 496, 561, 514, 501, 516, 510, 506, 517, 501, 496, 540, 517, 498, 525, 536, 513, 525, 531, 518, 474, 476, 525, 490, 484, 502, 527, 505, 547, 481, 499, 534, 522, 518, 504, 518, 495, 523, 518, 532, 542, 507, 506, 538, 486, 522, 525, 500, 524, 505, 528, 524, 534, 514, 521, 532, 454, 540, 501] payload_rows_per_dst=[10994, 10950, 10986, 10926, 10908, 10918, 10910, 10918] dup_rows_per_dst=[5430, 5416, 5140, 5564, 5398, 5444, 5592, 5578] combine_rows_per_src=[10994, 10950, 10986, 10926, 10908, 10918, 10910, 10918] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626933760 dispatch_meta_bytes=458968 combine_payload_bytes=626933760 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627392728 ingress_bytes=702431232 link_bytes=702431232 token_bytes=1254326488 weight_movement_bytes=75497472 total_nvlink_bytes=1329823960 weight_to_token_ratio=0.060190 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.907816768e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76514 dup_rows=43666 dup_rows_remote=38232 dedup_ratio=0.333145 router_tokens_per_expert=[491, 544, 528, 515, 494, 538, 514, 499, 509, 506, 519, 493, 528, 503, 561, 495, 529, 515, 495, 542, 506, 501, 520, 468, 543, 470, 503, 502, 522, 523, 534, 523, 512, 500, 527, 504, 526, 493, 544, 501, 497, 531, 525, 537, 532, 486, 527, 538, 521, 520, 523, 478, 525, 510, 470, 506, 480, 512, 554, 514, 553, 502, 517, 478, 488, 502, 512, 509, 543, 514, 459, 519, 519, 519, 465, 513, 469, 522, 514, 495, 487, 503, 508, 525, 503, 490, 503, 514, 500, 519, 498, 524, 508, 527, 481, 519, 528, 527, 512, 514, 527, 520, 549, 511, 489, 532, 557, 482, 481, 525, 498, 518, 515, 484, 525, 504, 520, 515, 529, 499, 525, 526, 497, 516, 492, 492, 526, 554] payload_rows_per_dst=[10972, 10938, 10960, 10892, 10802, 10920, 10922, 11000] dup_rows_per_dst=[5502, 5454, 5600, 5434, 5322, 5298, 5618, 5438] combine_rows_per_src=[10972, 10938, 10960, 10892, 10802, 10920, 10922, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87660 payload_rows_remote=76818 dup_rows=43412 dup_rows_remote=37960 dedup_ratio=0.331207 router_tokens_per_expert=[511, 532, 500, 491, 545, 546, 529, 505, 509, 496, 536, 487, 538, 517, 514, 500, 538, 473, 540, 510, 521, 501, 491, 540, 521, 511, 509, 513, 537, 496, 542, 520, 487, 538, 508, 545, 523, 499, 493, 532, 459, 514, 537, 533, 511, 554, 479, 523, 560, 472, 529, 538, 544, 468, 509, 478, 519, 529, 506, 537, 495, 500, 486, 520, 464, 522, 503, 528, 527, 516, 528, 519, 483, 495, 512, 484, 539, 493, 507, 527, 478, 509, 510, 481, 496, 525, 529, 508, 479, 543, 502, 526, 496, 513, 529, 500, 496, 534, 522, 489, 541, 518, 485, 537, 538, 505, 481, 486, 496, 524, 508, 498, 489, 487, 513, 514, 518, 498, 521, 511, 513, 522, 505, 500, 502, 494, 559, 517] payload_rows_per_dst=[11094, 10980, 10968, 11006, 10842, 10904, 10982, 10884] dup_rows_per_dst=[5418, 5546, 5502, 5374, 5452, 5344, 5334, 5442] combine_rows_per_src=[11094, 10980, 10968, 11006, 10842, 10904, 10982, 10884] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87662 payload_rows_remote=76836 dup_rows=43410 dup_rows_remote=38068 dedup_ratio=0.331192 router_tokens_per_expert=[528, 487, 521, 550, 508, 516, 499, 557, 527, 552, 537, 520, 496, 528, 476, 503, 497, 498, 506, 486, 539, 520, 529, 516, 492, 484, 505, 547, 548, 507, 519, 516, 509, 493, 486, 517, 504, 515, 518, 513, 520, 535, 527, 534, 549, 519, 503, 517, 532, 525, 514, 502, 536, 487, 480, 550, 518, 498, 480, 498, 520, 517, 488, 519, 531, 508, 523, 483, 516, 496, 549, 492, 537, 506, 487, 495, 525, 502, 530, 483, 501, 509, 522, 470, 521, 520, 482, 508, 527, 534, 516, 506, 531, 515, 516, 522, 521, 484, 515, 487, 499, 505, 479, 524, 527, 497, 502, 499, 516, 508, 486, 535, 495, 515, 538, 554, 468, 494, 469, 465, 538, 502, 537, 505, 525, 508, 510, 529] payload_rows_per_dst=[11054, 11048, 11018, 10912, 10880, 11008, 10826, 10916] dup_rows_per_dst=[5556, 5370, 5500, 5416, 5446, 5392, 5342, 5388] combine_rows_per_src=[11054, 11048, 11018, 10912, 10880, 11008, 10826, 10916] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=457944 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627473624 ingress_bytes=702513152 link_bytes=702513152 token_bytes=1254489304 weight_movement_bytes=75497472 total_nvlink_bytes=1329986776 weight_to_token_ratio=0.060182 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908855629e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=627441664 dispatch_meta_bytes=458840 combine_payload_bytes=627441664 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627900504 ingress_bytes=702939136 link_bytes=702939136 token_bytes=1255342168 weight_movement_bytes=75497472 total_nvlink_bytes=1330839640 weight_to_token_ratio=0.060141 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914257706e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626802688 dispatch_meta_bytes=458984 combine_payload_bytes=626802688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627261672 ingress_bytes=702300160 link_bytes=702300160 token_bytes=1254064360 weight_movement_bytes=75497472 total_nvlink_bytes=1329561832 weight_to_token_ratio=0.060202 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906154591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=629293056 dispatch_meta_bytes=459112 combine_payload_bytes=629293056 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629752168 ingress_bytes=704790528 link_bytes=704790528 token_bytes=1259045224 weight_movement_bytes=75497472 total_nvlink_bytes=1334542696 weight_to_token_ratio=0.059964 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.937735963e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87422 payload_rows_remote=76508 dup_rows=43650 dup_rows_remote=38100 dedup_ratio=0.333023 router_tokens_per_expert=[513, 506, 531, 521, 500, 540, 509, 487, 499, 526, 540, 493, 516, 515, 505, 531, 514, 484, 474, 502, 521, 505, 497, 512, 523, 555, 528, 472, 488, 509, 525, 505, 538, 539, 502, 495, 509, 476, 520, 500, 507, 529, 484, 526, 523, 477, 463, 540, 500, 490, 515, 514, 527, 500, 501, 520, 552, 506, 526, 507, 506, 531, 494, 533, 505, 534, 533, 521, 540, 505, 539, 534, 498, 513, 533, 486, 527, 524, 484, 478, 507, 518, 510, 527, 520, 494, 501, 489, 511, 500, 503, 537, 536, 529, 460, 494, 470, 525, 525, 506, 539, 520, 516, 489, 545, 510, 508, 518, 506, 579, 549, 509, 505, 500, 508, 530, 518, 495, 513, 551, 509, 503, 517, 452, 472, 509, 521, 533] payload_rows_per_dst=[10914, 10854, 10924, 10996, 10922, 10920, 10966, 10926] dup_rows_per_dst=[5550, 5374, 5332, 5598, 5586, 5464, 5400, 5346] combine_rows_per_src=[10914, 10854, 10924, 10996, 10922, 10920, 10966, 10926] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=629440512 dispatch_meta_bytes=459616 combine_payload_bytes=629440512 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=629900128 ingress_bytes=855932928 link_bytes=855932928 token_bytes=1259340640 weight_movement_bytes=226492416 total_nvlink_bytes=1485833056 weight_to_token_ratio=0.179850 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.085443434e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87310 payload_rows_remote=76282 dup_rows=43762 dup_rows_remote=38138 dedup_ratio=0.333878 router_tokens_per_expert=[529, 486, 527, 498, 555, 523, 523, 529, 484, 490, 561, 532, 525, 490, 541, 477, 529, 516, 505, 538, 511, 530, 505, 485, 471, 523, 500, 517, 481, 490, 519, 553, 537, 544, 517, 515, 516, 515, 528, 472, 488, 547, 536, 539, 531, 516, 515, 510, 515, 533, 525, 492, 508, 516, 481, 506, 525, 520, 538, 533, 495, 536, 527, 506, 540, 512, 501, 509, 496, 532, 483, 519, 536, 473, 511, 529, 497, 515, 510, 495, 520, 487, 499, 490, 499, 513, 495, 503, 542, 504, 537, 488, 516, 533, 500, 543, 534, 507, 528, 480, 498, 520, 543, 506, 491, 516, 503, 541, 487, 501, 523, 472, 489, 496, 497, 468, 515, 474, 497, 524, 525, 523, 487, 519, 492, 518, 505, 505] payload_rows_per_dst=[10946, 10880, 11028, 10948, 10858, 10950, 10866, 10834] dup_rows_per_dst=[5594, 5466, 5624, 5564, 5458, 5388, 5434, 5234] combine_rows_per_src=[10946, 10880, 11028, 10948, 10858, 10950, 10866, 10834] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=4 export_slots=4 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87432 payload_rows_remote=76420 dup_rows=43640 dup_rows_remote=38022 dedup_ratio=0.332947 router_tokens_per_expert=[507, 507, 518, 522, 526, 471, 470, 545, 526, 489, 510, 536, 511, 503, 508, 545, 473, 526, 559, 536, 527, 495, 493, 509, 545, 506, 518, 539, 522, 473, 562, 532, 493, 492, 502, 576, 522, 521, 482, 535, 524, 502, 542, 511, 503, 537, 539, 530, 523, 502, 507, 496, 516, 487, 504, 460, 528, 460, 513, 511, 508, 544, 514, 507, 522, 532, 517, 570, 499, 508, 494, 510, 484, 487, 528, 571, 495, 540, 512, 482, 483, 530, 481, 512, 543, 503, 462, 471, 495, 495, 502, 525, 499, 529, 515, 487, 512, 528, 520, 503, 482, 490, 481, 495, 498, 493, 562, 513, 482, 547, 554, 518, 520, 544, 523, 567, 513, 530, 480, 489, 529, 508, 499, 511, 456, 518, 488, 500] payload_rows_per_dst=[10980, 11012, 11040, 10894, 11032, 10704, 10888, 10882] dup_rows_per_dst=[5408, 5618, 5582, 5266, 5470, 5360, 5468, 5468] combine_rows_per_src=[10980, 11012, 11040, 10894, 11032, 10704, 10888, 10882] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=4 plans=1 dispatch_payload_bytes=626753536 dispatch_meta_bytes=458432 combine_payload_bytes=626753536 prefetch_bytes=0 reduce_bytes=301989888 egress_bytes=627211968 ingress_bytes=928743424 link_bytes=928743424 token_bytes=1253965504 weight_movement_bytes=301989888 total_nvlink_bytes=1555955392 weight_to_token_ratio=0.240828 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177777392e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87750 payload_rows_remote=76632 dup_rows=43322 dup_rows_remote=37778 dedup_ratio=0.330521 router_tokens_per_expert=[530, 544, 477, 504, 494, 496, 536, 518, 544, 521, 505, 507, 505, 526, 480, 564, 455, 517, 535, 520, 515, 488, 536, 544, 519, 505, 543, 503, 543, 505, 491, 502, 493, 519, 518, 506, 497, 523, 500, 503, 527, 495, 513, 505, 486, 518, 550, 493, 531, 530, 475, 504, 541, 520, 526, 529, 512, 506, 536, 482, 503, 512, 541, 507, 514, 516, 491, 559, 505, 526, 507, 494, 492, 551, 498, 496, 499, 491, 497, 517, 538, 519, 516, 532, 523, 501, 533, 514, 512, 543, 547, 529, 507, 490, 509, 516, 511, 515, 488, 491, 511, 458, 541, 521, 499, 501, 528, 506, 497, 523, 499, 544, 517, 542, 494, 482, 520, 529, 497, 535, 504, 492, 510, 487, 525, 516, 442, 456] payload_rows_per_dst=[10784, 10824, 10996, 11052, 10696, 11280, 11000, 11118] dup_rows_per_dst=[5082, 5306, 5318, 5738, 5054, 5800, 5480, 5544] combine_rows_per_src=[10784, 10824, 10996, 11052, 10696, 11280, 11000, 11118] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=624902144 dispatch_meta_bytes=457680 combine_payload_bytes=624902144 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625359824 ingress_bytes=775897088 link_bytes=775897088 token_bytes=1250261968 weight_movement_bytes=150994944 total_nvlink_bytes=1401256912 weight_to_token_ratio=0.120771 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.839467234e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626032640 dispatch_meta_bytes=457768 combine_payload_bytes=626032640 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=626490408 ingress_bytes=852525056 link_bytes=852525056 token_bytes=1252523048 weight_movement_bytes=226492416 total_nvlink_bytes=1479015464 weight_to_token_ratio=0.180829 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081121773e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87006 payload_rows_remote=76138 dup_rows=44066 dup_rows_remote=38624 dedup_ratio=0.336197 router_tokens_per_expert=[533, 533, 517, 467, 508, 505, 508, 506, 529, 552, 484, 539, 542, 476, 537, 506, 520, 489, 532, 511, 527, 535, 503, 499, 517, 476, 495, 507, 540, 479, 523, 502, 487, 531, 505, 534, 539, 561, 548, 533, 519, 512, 514, 520, 479, 497, 525, 525, 510, 508, 508, 512, 503, 528, 502, 479, 521, 532, 518, 487, 514, 521, 530, 482, 489, 515, 563, 495, 546, 523, 524, 508, 510, 525, 491, 535, 504, 523, 528, 541, 491, 485, 471, 543, 529, 487, 456, 489, 511, 525, 486, 498, 507, 478, 529, 520, 511, 532, 501, 528, 510, 515, 551, 528, 535, 508, 502, 506, 495, 500, 474, 506, 481, 496, 514, 509, 533, 508, 513, 518, 503, 518, 508, 517, 491, 463, 527, 529] payload_rows_per_dst=[10846, 10906, 10958, 10868, 10962, 10784, 10914, 10768] dup_rows_per_dst=[5638, 5404, 5700, 5442, 5678, 5226, 5490, 5488] combine_rows_per_src=[10846, 10906, 10958, 10868, 10962, 10784, 10914, 10768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87552 payload_rows_remote=76514 dup_rows=43520 dup_rows_remote=38150 dedup_ratio=0.332031 router_tokens_per_expert=[524, 495, 520, 492, 519, 505, 527, 497, 543, 477, 491, 479, 505, 481, 517, 500, 505, 509, 497, 472, 559, 489, 495, 517, 560, 486, 513, 519, 514, 481, 541, 519, 492, 507, 489, 522, 546, 471, 530, 488, 502, 495, 515, 498, 536, 520, 513, 450, 550, 535, 523, 512, 522, 519, 513, 489, 507, 489, 493, 528, 494, 555, 528, 501, 542, 537, 511, 505, 517, 512, 483, 526, 516, 531, 479, 535, 517, 531, 507, 504, 528, 507, 524, 505, 526, 516, 497, 473, 505, 529, 541, 511, 552, 529, 490, 471, 528, 521, 532, 552, 495, 504, 506, 491, 509, 493, 502, 513, 530, 525, 509, 497, 496, 523, 517, 523, 556, 525, 468, 534, 471, 515, 537, 531, 517, 526, 536, 517] payload_rows_per_dst=[10810, 10884, 10876, 10976, 10956, 11038, 11006, 11006] dup_rows_per_dst=[5334, 5468, 5272, 5540, 5550, 5370, 5408, 5578] combine_rows_per_src=[10810, 10884, 10876, 10976, 10956, 11038, 11006, 11006] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87234 payload_rows_remote=76254 dup_rows=43838 dup_rows_remote=38506 dedup_ratio=0.334457 router_tokens_per_expert=[528, 501, 504, 477, 532, 514, 519, 533, 502, 538, 518, 500, 521, 507, 528, 531, 496, 523, 536, 474, 462, 489, 517, 513, 494, 563, 517, 495, 553, 481, 513, 531, 535, 524, 532, 533, 497, 522, 512, 475, 496, 495, 505, 518, 524, 523, 535, 515, 490, 496, 496, 484, 544, 524, 545, 514, 484, 505, 505, 484, 478, 516, 534, 527, 501, 518, 499, 504, 495, 529, 463, 533, 537, 533, 518, 536, 480, 501, 507, 502, 491, 487, 503, 561, 564, 506, 523, 516, 501, 512, 523, 491, 531, 509, 525, 494, 477, 492, 473, 503, 501, 514, 485, 520, 496, 539, 540, 485, 529, 543, 479, 553, 512, 530, 537, 523, 526, 516, 516, 536, 472, 504, 517, 507, 536, 483, 541, 481] payload_rows_per_dst=[10776, 10884, 10916, 10868, 10980, 10974, 10836, 11000] dup_rows_per_dst=[5730, 5430, 5566, 5384, 5332, 5500, 5422, 5474] combine_rows_per_src=[10776, 10884, 10916, 10868, 10980, 10974, 10836, 11000] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87440 payload_rows_remote=76448 dup_rows=43632 dup_rows_remote=38114 dedup_ratio=0.332886 router_tokens_per_expert=[509, 500, 495, 521, 508, 485, 538, 528, 493, 525, 538, 533, 504, 475, 554, 511, 550, 475, 490, 503, 510, 546, 552, 526, 499, 510, 511, 495, 500, 549, 498, 523, 531, 536, 490, 473, 493, 547, 562, 491, 504, 504, 556, 504, 514, 458, 490, 510, 519, 534, 513, 515, 482, 507, 511, 485, 534, 508, 495, 544, 502, 500, 523, 489, 505, 518, 468, 497, 552, 505, 534, 553, 543, 520, 508, 542, 506, 518, 524, 512, 495, 509, 521, 529, 496, 547, 512, 533, 506, 514, 490, 508, 483, 502, 514, 526, 525, 527, 528, 519, 531, 543, 537, 507, 548, 506, 495, 505, 490, 530, 475, 489, 512, 499, 473, 491, 518, 472, 492, 509, 498, 499, 500, 472, 528, 487, 539, 524] payload_rows_per_dst=[10968, 10912, 10884, 10876, 11072, 10982, 10992, 10754] dup_rows_per_dst=[5466, 5562, 5442, 5446, 5538, 5388, 5518, 5272] combine_rows_per_src=[10968, 10912, 10884, 10876, 11072, 10982, 10992, 10754] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=457640 combine_payload_bytes=627769344 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628226984 ingress_bytes=703266816 link_bytes=703266816 token_bytes=1255996328 weight_movement_bytes=75497472 total_nvlink_bytes=1331493800 weight_to_token_ratio=0.060110 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918413150e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=623722496 dispatch_meta_bytes=459048 combine_payload_bytes=623722496 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624181544 ingress_bytes=699219968 link_bytes=699219968 token_bytes=1247904040 weight_movement_bytes=75497472 total_nvlink_bytes=1323401512 weight_to_token_ratio=0.060499 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.867093421e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626802688 dispatch_meta_bytes=458656 combine_payload_bytes=626802688 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627261344 ingress_bytes=702300160 link_bytes=702300160 token_bytes=1254064032 weight_movement_bytes=75497472 total_nvlink_bytes=1329561504 weight_to_token_ratio=0.060202 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906154591e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=624672768 dispatch_meta_bytes=459040 combine_payload_bytes=624672768 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625131808 ingress_bytes=775667712 link_bytes=775667712 token_bytes=1249804576 weight_movement_bytes=150994944 total_nvlink_bytes=1400799520 weight_to_token_ratio=0.120815 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.836558423e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=626262016 dispatch_meta_bytes=458248 combine_payload_bytes=626262016 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626720264 ingress_bytes=701759488 link_bytes=701759488 token_bytes=1252982280 weight_movement_bytes=75497472 total_nvlink_bytes=1328479752 weight_to_token_ratio=0.060254 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.899298109e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87450 payload_rows_remote=76404 dup_rows=43622 dup_rows_remote=37958 dedup_ratio=0.332809 router_tokens_per_expert=[506, 578, 481, 507, 527, 544, 535, 517, 502, 541, 539, 502, 496, 489, 516, 523, 472, 506, 539, 494, 527, 508, 567, 508, 510, 501, 537, 496, 541, 522, 544, 486, 496, 494, 530, 493, 509, 493, 516, 479, 521, 506, 512, 474, 479, 483, 496, 506, 516, 506, 513, 489, 535, 504, 525, 455, 494, 523, 539, 495, 466, 513, 528, 504, 486, 517, 525, 474, 522, 477, 483, 541, 544, 524, 517, 502, 558, 529, 507, 512, 512, 524, 469, 520, 542, 504, 532, 514, 528, 477, 525, 535, 502, 503, 499, 478, 539, 482, 486, 543, 484, 564, 514, 505, 493, 546, 483, 515, 485, 529, 490, 517, 520, 517, 508, 508, 539, 557, 533, 477, 508, 497, 551, 545, 540, 510, 515, 501] payload_rows_per_dst=[11046, 10944, 10772, 10898, 11030, 10824, 10870, 11066] dup_rows_per_dst=[5664, 5572, 5098, 5312, 5406, 5504, 5480, 5586] combine_rows_per_src=[11046, 10944, 10772, 10898, 11030, 10824, 10870, 11066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87306 payload_rows_remote=76388 dup_rows=43766 dup_rows_remote=38186 dedup_ratio=0.333908 router_tokens_per_expert=[489, 506, 523, 543, 546, 518, 510, 502, 509, 511, 537, 528, 433, 509, 539, 516, 538, 482, 512, 526, 494, 461, 502, 505, 541, 515, 518, 520, 525, 541, 547, 515, 536, 516, 506, 482, 525, 497, 510, 528, 536, 519, 536, 514, 517, 483, 535, 509, 530, 547, 518, 528, 528, 504, 501, 488, 484, 495, 493, 489, 497, 485, 485, 532, 503, 518, 514, 503, 509, 509, 517, 531, 472, 556, 523, 507, 528, 478, 554, 477, 496, 495, 477, 469, 514, 547, 486, 545, 524, 534, 528, 492, 500, 503, 552, 529, 499, 513, 483, 510, 486, 545, 522, 543, 517, 482, 501, 550, 529, 485, 514, 546, 498, 527, 503, 513, 520, 508, 542, 547, 519, 455, 488, 493, 495, 526, 484, 489] payload_rows_per_dst=[10886, 10978, 10918, 10868, 10888, 10920, 10980, 10868] dup_rows_per_dst=[5552, 5506, 5580, 5340, 5510, 5462, 5470, 5346] combine_rows_per_src=[10886, 10978, 10918, 10868, 10888, 10920, 10980, 10868] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87506 payload_rows_remote=76636 dup_rows=43566 dup_rows_remote=38304 dedup_ratio=0.332382 router_tokens_per_expert=[500, 534, 499, 482, 496, 524, 542, 504, 497, 476, 521, 534, 558, 529, 492, 480, 523, 493, 529, 481, 521, 482, 554, 479, 479, 523, 510, 526, 503, 454, 499, 510, 531, 508, 531, 510, 517, 531, 551, 488, 534, 531, 509, 507, 534, 515, 521, 511, 497, 498, 505, 541, 520, 516, 508, 514, 532, 506, 493, 474, 490, 511, 487, 477, 516, 513, 508, 503, 514, 526, 538, 495, 502, 511, 502, 527, 527, 523, 543, 520, 480, 515, 540, 534, 528, 525, 516, 528, 510, 544, 468, 515, 538, 495, 528, 487, 484, 492, 539, 518, 506, 540, 514, 481, 524, 502, 510, 528, 546, 507, 538, 490, 478, 475, 487, 472, 495, 548, 567, 519, 517, 528, 530, 470, 528, 557, 473, 522] payload_rows_per_dst=[10886, 10870, 11102, 10750, 10966, 11042, 10962, 10928] dup_rows_per_dst=[5450, 5262, 5556, 5388, 5570, 5460, 5476, 5404] combine_rows_per_src=[10886, 10870, 11102, 10750, 10966, 11042, 10962, 10928] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=5 plans=1 dispatch_payload_bytes=625901568 dispatch_meta_bytes=457448 combine_payload_bytes=625901568 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626359016 ingress_bytes=852393984 link_bytes=852393984 token_bytes=1252260584 weight_movement_bytes=226492416 total_nvlink_bytes=1478753000 weight_to_token_ratio=0.180867 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080955555e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625770496 dispatch_meta_bytes=458296 combine_payload_bytes=625770496 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626228792 ingress_bytes=852262912 link_bytes=852262912 token_bytes=1251999288 weight_movement_bytes=226492416 total_nvlink_bytes=1478491704 weight_to_token_ratio=0.180905 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080789337e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87584 payload_rows_remote=76294 dup_rows=43488 dup_rows_remote=37704 dedup_ratio=0.331787 router_tokens_per_expert=[530, 518, 501, 519, 525, 527, 550, 520, 497, 498, 508, 503, 513, 514, 535, 524, 461, 516, 502, 564, 508, 565, 528, 524, 520, 521, 493, 515, 467, 511, 532, 494, 506, 505, 498, 544, 546, 565, 532, 508, 495, 496, 501, 476, 531, 495, 509, 494, 505, 487, 507, 514, 472, 481, 544, 496, 509, 496, 506, 533, 503, 535, 543, 486, 495, 516, 521, 481, 494, 531, 511, 539, 507, 497, 546, 491, 521, 512, 549, 497, 486, 535, 528, 485, 497, 507, 517, 529, 505, 484, 499, 537, 504, 546, 480, 527, 560, 503, 502, 468, 520, 518, 505, 538, 523, 489, 512, 511, 505, 498, 481, 542, 505, 505, 514, 491, 529, 504, 492, 539, 533, 506, 482, 505, 481, 497, 515, 568] payload_rows_per_dst=[11010, 11026, 10624, 10916, 10908, 10942, 10868, 11290] dup_rows_per_dst=[5554, 5406, 5004, 5318, 5508, 5428, 5486, 5784] combine_rows_per_src=[11010, 11026, 10624, 10916, 10908, 10942, 10868, 11290] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=459760 combine_payload_bytes=627802112 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=628261872 ingress_bytes=854294528 link_bytes=854294528 token_bytes=1256063984 weight_movement_bytes=226492416 total_nvlink_bytes=1482556400 weight_to_token_ratio=0.180319 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.042279 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.083365713e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87510 payload_rows_remote=76536 dup_rows=43562 dup_rows_remote=38196 dedup_ratio=0.332352 router_tokens_per_expert=[479, 514, 510, 533, 525, 497, 512, 513, 539, 509, 473, 532, 548, 488, 520, 531, 501, 533, 466, 511, 523, 501, 534, 491, 544, 491, 486, 537, 509, 475, 514, 488, 494, 510, 517, 500, 535, 527, 480, 486, 493, 515, 530, 552, 490, 516, 527, 520, 508, 466, 549, 542, 513, 493, 494, 499, 516, 539, 515, 537, 476, 498, 509, 478, 520, 549, 508, 472, 532, 513, 510, 517, 507, 510, 472, 535, 514, 491, 527, 488, 548, 516, 513, 558, 491, 531, 493, 523, 525, 486, 528, 501, 525, 486, 443, 503, 520, 502, 468, 525, 488, 515, 535, 531, 543, 494, 513, 513, 576, 497, 514, 496, 494, 521, 516, 529, 494, 544, 487, 550, 516, 507, 510, 583, 555, 493, 520, 501] payload_rows_per_dst=[10968, 10910, 10860, 10954, 10838, 10974, 10940, 11066] dup_rows_per_dst=[5478, 5298, 5524, 5310, 5492, 5366, 5520, 5574] combine_rows_per_src=[10968, 10910, 10860, 10954, 10838, 10974, 10940, 11066] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87538 payload_rows_remote=76552 dup_rows=43534 dup_rows_remote=38366 dedup_ratio=0.332138 router_tokens_per_expert=[513, 496, 507, 526, 535, 496, 502, 482, 520, 522, 514, 510, 535, 509, 528, 527, 502, 515, 453, 515, 466, 542, 492, 528, 488, 526, 510, 493, 528, 551, 521, 502, 479, 519, 547, 519, 514, 527, 511, 524, 495, 504, 518, 521, 545, 529, 532, 531, 475, 525, 519, 498, 518, 525, 498, 498, 477, 533, 502, 490, 506, 497, 510, 506, 507, 526, 512, 519, 477, 484, 530, 523, 480, 523, 517, 471, 481, 537, 517, 534, 522, 538, 536, 523, 508, 490, 489, 503, 506, 494, 504, 518, 482, 507, 516, 522, 546, 534, 478, 503, 545, 523, 528, 522, 554, 530, 523, 506, 519, 491, 559, 513, 505, 475, 497, 515, 510, 511, 496, 511, 525, 500, 515, 492, 522, 499, 536, 511] payload_rows_per_dst=[10874, 10924, 10998, 10986, 10850, 10866, 11126, 10914] dup_rows_per_dst=[5570, 5340, 5632, 5168, 5426, 5450, 5622, 5326] combine_rows_per_src=[10874, 10924, 10998, 10986, 10850, 10866, 11126, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87532 payload_rows_remote=76650 dup_rows=43540 dup_rows_remote=38278 dedup_ratio=0.332184 router_tokens_per_expert=[487, 508, 470, 530, 498, 459, 554, 486, 552, 508, 493, 500, 462, 546, 542, 495, 511, 497, 546, 536, 479, 530, 539, 530, 498, 493, 508, 517, 538, 496, 503, 508, 508, 494, 500, 537, 542, 521, 529, 513, 501, 505, 514, 527, 529, 515, 526, 534, 472, 514, 539, 513, 486, 555, 526, 479, 510, 503, 469, 539, 506, 518, 503, 531, 517, 511, 541, 490, 497, 489, 481, 513, 549, 509, 477, 508, 518, 468, 522, 482, 546, 548, 494, 536, 505, 520, 526, 523, 516, 519, 537, 500, 515, 544, 530, 461, 499, 526, 495, 515, 507, 533, 485, 496, 494, 496, 557, 539, 568, 517, 523, 523, 509, 537, 520, 468, 534, 504, 529, 520, 496, 480, 476, 476, 513, 523, 487, 522] payload_rows_per_dst=[10842, 10966, 11038, 10850, 10882, 11034, 11072, 10848] dup_rows_per_dst=[5338, 5492, 5552, 5476, 5262, 5606, 5474, 5340] combine_rows_per_src=[10842, 10966, 11038, 10850, 10882, 11034, 11072, 10848] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87374 payload_rows_remote=76464 dup_rows=43698 dup_rows_remote=38246 dedup_ratio=0.333389 router_tokens_per_expert=[482, 531, 470, 509, 498, 513, 520, 527, 482, 476, 496, 504, 471, 514, 493, 528, 531, 510, 516, 511, 540, 538, 511, 547, 470, 523, 517, 509, 530, 507, 517, 521, 506, 502, 465, 533, 497, 489, 532, 519, 509, 526, 522, 512, 497, 532, 575, 477, 506, 528, 539, 529, 474, 486, 519, 541, 457, 559, 533, 522, 513, 552, 529, 532, 546, 546, 565, 508, 503, 502, 538, 509, 485, 514, 515, 487, 478, 515, 493, 550, 521, 525, 558, 500, 471, 484, 570, 543, 490, 484, 513, 492, 502, 505, 482, 520, 538, 470, 553, 490, 496, 538, 538, 462, 549, 529, 510, 491, 507, 501, 504, 505, 503, 526, 520, 496, 490, 543, 496, 488, 518, 448, 542, 469, 498, 500, 527, 553] payload_rows_per_dst=[10834, 10970, 10884, 11032, 11018, 10840, 10910, 10886] dup_rows_per_dst=[5194, 5626, 5502, 5606, 5490, 5480, 5452, 5348] combine_rows_per_src=[10834, 10970, 10884, 11032, 11018, 10840, 10910, 10886] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=625000448 dispatch_meta_bytes=455992 combine_payload_bytes=625000448 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625456440 ingress_bytes=700497920 link_bytes=700497920 token_bytes=1250456888 weight_movement_bytes=75497472 total_nvlink_bytes=1325954360 weight_to_token_ratio=0.060376 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.883299651e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626982912 dispatch_meta_bytes=458928 combine_payload_bytes=626982912 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627441840 ingress_bytes=702480384 link_bytes=702480384 token_bytes=1254424752 weight_movement_bytes=75497472 total_nvlink_bytes=1329922224 weight_to_token_ratio=0.060185 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908440085e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627916800 dispatch_meta_bytes=459712 combine_payload_bytes=627916800 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628376512 ingress_bytes=703414272 link_bytes=703414272 token_bytes=1256293312 weight_movement_bytes=75497472 total_nvlink_bytes=1331790784 weight_to_token_ratio=0.060095 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.920283099e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=627113984 dispatch_meta_bytes=459672 combine_payload_bytes=627113984 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627573656 ingress_bytes=702611456 link_bytes=702611456 token_bytes=1254687640 weight_movement_bytes=75497472 total_nvlink_bytes=1330185112 weight_to_token_ratio=0.060172 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910102262e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626393088 dispatch_meta_bytes=458840 combine_payload_bytes=626393088 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626851928 ingress_bytes=852885504 link_bytes=852885504 token_bytes=1253245016 weight_movement_bytes=226492416 total_nvlink_bytes=1479737432 weight_to_token_ratio=0.180725 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081578872e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87370 payload_rows_remote=76442 dup_rows=43702 dup_rows_remote=38370 dedup_ratio=0.333420 router_tokens_per_expert=[498, 487, 490, 523, 519, 524, 519, 526, 528, 529, 521, 531, 502, 500, 521, 490, 511, 524, 517, 463, 540, 504, 530, 504, 551, 517, 511, 500, 489, 475, 510, 492, 507, 501, 498, 516, 498, 490, 498, 479, 497, 549, 545, 487, 467, 570, 526, 502, 516, 532, 542, 516, 562, 497, 563, 479, 510, 547, 547, 532, 505, 515, 507, 543, 565, 523, 519, 496, 505, 523, 493, 480, 509, 520, 542, 515, 526, 520, 539, 517, 510, 514, 516, 514, 522, 520, 553, 505, 475, 501, 520, 493, 498, 482, 483, 521, 523, 475, 502, 504, 492, 510, 531, 538, 500, 501, 518, 496, 535, 500, 484, 497, 492, 496, 516, 476, 517, 489, 467, 512, 518, 523, 529, 565, 498, 488, 512, 524] payload_rows_per_dst=[10908, 10832, 10928, 11126, 10990, 10820, 10864, 10902] dup_rows_per_dst=[5508, 5444, 5332, 5700, 5594, 5434, 5348, 5342] combine_rows_per_src=[10908, 10832, 10928, 11126, 10990, 10820, 10864, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87660 payload_rows_remote=76484 dup_rows=43412 dup_rows_remote=37952 dedup_ratio=0.331207 router_tokens_per_expert=[503, 513, 508, 508, 527, 505, 512, 574, 519, 512, 491, 508, 563, 505, 483, 490, 497, 526, 499, 513, 506, 534, 513, 539, 511, 527, 557, 516, 509, 574, 514, 511, 487, 502, 484, 505, 466, 544, 538, 521, 505, 542, 511, 492, 507, 478, 526, 541, 470, 504, 523, 520, 557, 476, 519, 523, 497, 542, 505, 489, 519, 511, 508, 520, 506, 506, 540, 495, 517, 495, 513, 544, 497, 499, 508, 525, 508, 481, 506, 526, 479, 508, 488, 502, 507, 521, 504, 522, 510, 527, 458, 524, 501, 516, 517, 517, 549, 497, 532, 547, 512, 495, 473, 554, 476, 490, 532, 510, 507, 549, 548, 524, 490, 529, 496, 517, 506, 560, 456, 484, 522, 484, 499, 515, 533, 488, 500, 496] payload_rows_per_dst=[11176, 11028, 10906, 11246, 11004, 10886, 10632, 10782] dup_rows_per_dst=[5460, 5470, 5392, 5864, 5460, 5316, 5082, 5368] combine_rows_per_src=[11176, 11028, 10906, 11246, 11004, 10886, 10632, 10782] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87678 payload_rows_remote=76618 dup_rows=43394 dup_rows_remote=38134 dedup_ratio=0.331070 router_tokens_per_expert=[516, 502, 474, 540, 508, 521, 551, 536, 495, 483, 472, 511, 555, 514, 491, 494, 478, 520, 541, 526, 509, 515, 498, 510, 515, 508, 542, 521, 478, 501, 479, 519, 503, 511, 465, 507, 517, 528, 514, 541, 557, 505, 518, 508, 505, 478, 513, 508, 519, 519, 519, 523, 525, 495, 547, 510, 488, 509, 514, 518, 510, 474, 502, 542, 506, 517, 531, 513, 512, 493, 509, 480, 510, 532, 512, 510, 527, 483, 522, 512, 500, 488, 520, 482, 489, 496, 500, 540, 525, 491, 496, 510, 511, 544, 500, 511, 494, 520, 524, 540, 493, 535, 521, 497, 534, 560, 532, 508, 517, 539, 524, 495, 488, 489, 505, 529, 527, 513, 491, 507, 528, 540, 507, 519, 522, 490, 512, 549] payload_rows_per_dst=[10892, 11060, 10936, 10874, 10920, 10922, 11098, 10976] dup_rows_per_dst=[5434, 5260, 5420, 5554, 5418, 5284, 5568, 5456] combine_rows_per_src=[10892, 11060, 10936, 10874, 10920, 10922, 11098, 10976] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626212864 dispatch_meta_bytes=459248 combine_payload_bytes=626212864 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626672112 ingress_bytes=852705280 link_bytes=852705280 token_bytes=1252884976 weight_movement_bytes=226492416 total_nvlink_bytes=1479377392 weight_to_token_ratio=0.180777 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081350322e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87452 payload_rows_remote=76396 dup_rows=43620 dup_rows_remote=38056 dedup_ratio=0.332794 router_tokens_per_expert=[473, 488, 517, 559, 532, 541, 492, 542, 523, 502, 505, 465, 525, 513, 531, 502, 551, 491, 516, 483, 534, 532, 522, 502, 500, 488, 507, 499, 515, 508, 476, 495, 524, 510, 508, 485, 487, 482, 548, 520, 561, 532, 470, 466, 505, 509, 542, 502, 535, 457, 520, 547, 487, 533, 495, 515, 554, 499, 494, 501, 511, 555, 520, 503, 502, 511, 495, 505, 503, 531, 490, 524, 470, 489, 513, 489, 469, 538, 491, 546, 470, 515, 541, 558, 493, 516, 530, 473, 541, 485, 516, 537, 549, 495, 514, 536, 533, 540, 503, 509, 543, 503, 496, 510, 477, 525, 531, 527, 464, 505, 532, 495, 503, 547, 477, 519, 543, 521, 550, 485, 483, 536, 515, 527, 558, 529, 520, 489] payload_rows_per_dst=[10862, 10840, 10712, 10986, 10994, 10990, 11012, 11056] dup_rows_per_dst=[5558, 5398, 5150, 5466, 5562, 5548, 5374, 5564] combine_rows_per_src=[10862, 10840, 10712, 10986, 10994, 10990, 11012, 11056] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=6 plans=1 dispatch_payload_bytes=626556928 dispatch_meta_bytes=457744 combine_payload_bytes=626556928 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627014672 ingress_bytes=702054400 link_bytes=702054400 token_bytes=1253571600 weight_movement_bytes=75497472 total_nvlink_bytes=1329069072 weight_to_token_ratio=0.060226 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.903038008e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627654656 dispatch_meta_bytes=459008 combine_payload_bytes=627654656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628113664 ingress_bytes=703152128 link_bytes=703152128 token_bytes=1255768320 weight_movement_bytes=75497472 total_nvlink_bytes=1331265792 weight_to_token_ratio=0.060121 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916958744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87344 payload_rows_remote=76516 dup_rows=43728 dup_rows_remote=38354 dedup_ratio=0.333618 router_tokens_per_expert=[518, 475, 532, 473, 496, 515, 513, 537, 522, 460, 495, 487, 526, 542, 516, 536, 522, 515, 520, 528, 502, 519, 514, 492, 506, 519, 504, 512, 518, 493, 531, 481, 534, 480, 504, 509, 524, 514, 492, 503, 505, 516, 507, 523, 517, 485, 578, 533, 499, 498, 512, 567, 496, 525, 531, 521, 482, 538, 508, 483, 501, 485, 503, 529, 534, 476, 498, 502, 511, 513, 484, 511, 503, 517, 509, 539, 523, 520, 503, 506, 554, 478, 532, 473, 493, 505, 517, 475, 494, 495, 541, 519, 508, 525, 474, 518, 488, 548, 538, 551, 484, 485, 525, 523, 552, 553, 513, 515, 531, 494, 514, 500, 537, 509, 519, 516, 514, 520, 512, 528, 514, 510, 508, 529, 481, 477, 550, 527] payload_rows_per_dst=[10902, 10916, 10912, 10866, 10842, 10828, 11016, 11062] dup_rows_per_dst=[5384, 5436, 5536, 5490, 5456, 5374, 5612, 5440] combine_rows_per_src=[10902, 10916, 10912, 10866, 10842, 10828, 11016, 11062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87434 payload_rows_remote=76618 dup_rows=43638 dup_rows_remote=38166 dedup_ratio=0.332932 router_tokens_per_expert=[542, 496, 548, 511, 511, 490, 523, 526, 535, 509, 505, 533, 497, 504, 513, 521, 491, 519, 555, 507, 488, 510, 525, 503, 525, 517, 521, 533, 499, 487, 517, 485, 495, 523, 522, 510, 547, 524, 535, 519, 529, 521, 538, 474, 490, 536, 508, 512, 471, 519, 467, 487, 506, 507, 482, 527, 501, 486, 518, 504, 491, 478, 503, 476, 505, 483, 531, 525, 533, 512, 519, 487, 526, 496, 481, 497, 545, 525, 481, 498, 544, 531, 515, 507, 545, 530, 516, 512, 508, 503, 523, 491, 498, 482, 516, 487, 502, 528, 496, 492, 547, 504, 510, 520, 501, 543, 478, 511, 506, 540, 486, 546, 547, 506, 513, 550, 496, 532, 518, 538, 529, 495, 534, 510, 491, 507, 547, 509] payload_rows_per_dst=[11026, 10886, 11016, 10718, 10816, 10966, 10944, 11062] dup_rows_per_dst=[5502, 5478, 5550, 5128, 5472, 5450, 5476, 5582] combine_rows_per_src=[11026, 10886, 11016, 10718, 10816, 10966, 10944, 11062] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87624 payload_rows_remote=76632 dup_rows=43448 dup_rows_remote=37930 dedup_ratio=0.331482 router_tokens_per_expert=[505, 507, 517, 548, 497, 532, 519, 501, 507, 482, 500, 495, 534, 532, 509, 486, 461, 518, 483, 501, 530, 568, 524, 501, 510, 516, 524, 527, 506, 524, 491, 498, 521, 559, 495, 562, 536, 507, 490, 502, 515, 477, 529, 497, 544, 516, 513, 474, 492, 523, 521, 487, 511, 545, 546, 518, 500, 539, 502, 515, 521, 502, 490, 543, 518, 489, 531, 488, 493, 517, 503, 542, 522, 506, 526, 454, 535, 525, 484, 542, 532, 528, 485, 531, 511, 487, 510, 465, 531, 503, 541, 538, 541, 498, 496, 501, 488, 531, 550, 487, 557, 533, 486, 528, 543, 501, 510, 504, 475, 483, 517, 488, 476, 525, 529, 498, 505, 488, 462, 505, 506, 476, 519, 527, 497, 560, 532, 532] payload_rows_per_dst=[10956, 10954, 10932, 10992, 11054, 10940, 10936, 10860] dup_rows_per_dst=[5386, 5410, 5542, 5518, 5296, 5456, 5426, 5414] combine_rows_per_src=[10956, 10954, 10932, 10992, 11054, 10940, 10936, 10860] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87560 payload_rows_remote=76612 dup_rows=43512 dup_rows_remote=38092 dedup_ratio=0.331970 router_tokens_per_expert=[482, 467, 514, 506, 548, 472, 519, 565, 497, 494, 512, 518, 492, 532, 529, 511, 488, 491, 505, 509, 476, 514, 524, 499, 460, 511, 506, 491, 511, 497, 514, 540, 550, 519, 527, 548, 517, 525, 525, 495, 496, 530, 572, 548, 494, 486, 554, 510, 483, 537, 509, 524, 493, 536, 542, 511, 487, 513, 517, 487, 534, 516, 496, 515, 544, 522, 518, 516, 522, 492, 501, 511, 514, 494, 501, 497, 490, 516, 526, 519, 520, 509, 500, 501, 550, 481, 535, 498, 508, 504, 577, 519, 495, 498, 535, 499, 513, 509, 491, 500, 526, 525, 500, 507, 535, 493, 509, 502, 491, 532, 499, 552, 478, 509, 514, 512, 517, 485, 510, 509, 575, 546, 513, 496, 477, 504, 502, 503] payload_rows_per_dst=[10902, 10854, 11058, 11024, 10860, 11014, 10948, 10900] dup_rows_per_dst=[5414, 5218, 5734, 5376, 5506, 5444, 5420, 5400] combine_rows_per_src=[10902, 10854, 11058, 11024, 10860, 11014, 10948, 10900] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=625836032 dispatch_meta_bytes=457808 combine_payload_bytes=625836032 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626293840 ingress_bytes=852328448 link_bytes=852328448 token_bytes=1252129872 weight_movement_bytes=226492416 total_nvlink_bytes=1478622288 weight_to_token_ratio=0.180886 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080872446e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=626819072 dispatch_meta_bytes=459480 combine_payload_bytes=626819072 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627278552 ingress_bytes=702316544 link_bytes=702316544 token_bytes=1254097624 weight_movement_bytes=75497472 total_nvlink_bytes=1329595096 weight_to_token_ratio=0.060201 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906362363e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627654656 dispatch_meta_bytes=459136 combine_payload_bytes=627654656 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628113792 ingress_bytes=703152128 link_bytes=703152128 token_bytes=1255768448 weight_movement_bytes=75497472 total_nvlink_bytes=1331265920 weight_to_token_ratio=0.060121 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.916958744e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627769344 dispatch_meta_bytes=458248 combine_payload_bytes=627769344 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628227592 ingress_bytes=778764288 link_bytes=778764288 token_bytes=1255996936 weight_movement_bytes=150994944 total_nvlink_bytes=1406991880 weight_to_token_ratio=0.120219 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.047794 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.875827365e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=627605504 dispatch_meta_bytes=458816 combine_payload_bytes=627605504 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=628064320 ingress_bytes=778600448 link_bytes=778600448 token_bytes=1255669824 weight_movement_bytes=150994944 total_nvlink_bytes=1406664768 weight_to_token_ratio=0.120251 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.873749644e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87586 payload_rows_remote=76546 dup_rows=43486 dup_rows_remote=37928 dedup_ratio=0.331772 router_tokens_per_expert=[496, 502, 536, 496, 495, 515, 519, 490, 523, 517, 536, 495, 503, 531, 507, 495, 545, 492, 477, 501, 504, 500, 543, 487, 538, 505, 475, 527, 533, 487, 464, 495, 500, 509, 536, 512, 529, 521, 490, 536, 523, 538, 517, 531, 505, 510, 530, 512, 531, 481, 504, 555, 464, 507, 490, 499, 511, 534, 498, 504, 510, 532, 486, 508, 480, 490, 488, 522, 563, 517, 521, 515, 525, 531, 544, 534, 500, 522, 499, 474, 534, 506, 544, 525, 483, 468, 528, 505, 505, 548, 519, 534, 507, 546, 494, 557, 504, 512, 499, 533, 485, 514, 512, 483, 485, 506, 534, 519, 490, 488, 525, 506, 539, 464, 465, 524, 512, 522, 484, 525, 549, 506, 563, 511, 515, 519, 530, 543] payload_rows_per_dst=[10898, 10786, 11040, 10812, 11028, 11080, 10848, 11094] dup_rows_per_dst=[5414, 5360, 5558, 5416, 5422, 5526, 5342, 5448] combine_rows_per_src=[10898, 10786, 11040, 10812, 11028, 11080, 10848, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87236 payload_rows_remote=76348 dup_rows=43836 dup_rows_remote=38358 dedup_ratio=0.334442 router_tokens_per_expert=[551, 503, 500, 522, 518, 517, 508, 517, 497, 490, 453, 491, 507, 479, 533, 531, 510, 508, 503, 507, 504, 499, 515, 497, 511, 511, 510, 525, 485, 517, 568, 513, 546, 519, 515, 492, 511, 488, 487, 532, 525, 514, 539, 519, 528, 535, 487, 543, 535, 530, 498, 507, 496, 523, 544, 503, 528, 528, 476, 501, 537, 466, 497, 486, 521, 499, 484, 501, 495, 507, 520, 486, 542, 514, 509, 496, 508, 501, 493, 521, 487, 507, 515, 540, 516, 515, 519, 499, 504, 511, 551, 519, 505, 495, 509, 535, 539, 494, 559, 520, 493, 484, 527, 516, 484, 529, 492, 533, 516, 493, 518, 488, 542, 509, 509, 507, 549, 508, 486, 523, 509, 493, 494, 513, 576, 518, 535, 521] payload_rows_per_dst=[10870, 10888, 10958, 10850, 10858, 10982, 10822, 11008] dup_rows_per_dst=[5364, 5478, 5602, 5460, 5336, 5472, 5548, 5576] combine_rows_per_src=[10870, 10888, 10958, 10850, 10858, 10982, 10822, 11008] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87206 payload_rows_remote=76258 dup_rows=43866 dup_rows_remote=38376 dedup_ratio=0.334671 router_tokens_per_expert=[524, 524, 515, 509, 519, 511, 507, 510, 477, 510, 508, 516, 470, 525, 529, 527, 496, 522, 483, 494, 533, 524, 489, 551, 478, 540, 474, 465, 518, 507, 524, 518, 507, 525, 510, 526, 488, 513, 530, 480, 526, 535, 492, 524, 490, 513, 486, 509, 543, 492, 491, 528, 482, 478, 530, 512, 515, 519, 486, 521, 527, 512, 520, 512, 506, 550, 501, 506, 502, 507, 489, 531, 509, 511, 501, 492, 538, 521, 510, 472, 542, 489, 525, 518, 483, 537, 518, 500, 496, 545, 489, 526, 532, 507, 517, 521, 493, 493, 493, 534, 523, 552, 490, 525, 535, 560, 479, 505, 505, 544, 562, 544, 497, 565, 536, 530, 501, 525, 509, 489, 540, 509, 533, 503, 474, 505, 472, 501] payload_rows_per_dst=[10948, 11092, 10830, 10866, 10932, 10980, 10688, 10870] dup_rows_per_dst=[5490, 5994, 5484, 5388, 5360, 5622, 5342, 5186] combine_rows_per_src=[10948, 11092, 10830, 10866, 10932, 10980, 10688, 10870] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627064832 dispatch_meta_bytes=457896 combine_payload_bytes=627064832 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627522728 ingress_bytes=778059776 link_bytes=778059776 token_bytes=1254587560 weight_movement_bytes=150994944 total_nvlink_bytes=1405582504 weight_to_token_ratio=0.120354 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866893162e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87690 payload_rows_remote=76596 dup_rows=43382 dup_rows_remote=38054 dedup_ratio=0.330978 router_tokens_per_expert=[500, 502, 525, 518, 511, 519, 500, 553, 528, 512, 489, 524, 552, 527, 505, 507, 491, 517, 537, 547, 511, 497, 511, 542, 522, 531, 507, 524, 455, 534, 476, 494, 490, 498, 522, 505, 524, 533, 493, 534, 523, 515, 504, 516, 539, 497, 515, 500, 490, 509, 511, 562, 507, 502, 513, 494, 520, 526, 506, 479, 476, 511, 508, 495, 473, 504, 545, 523, 508, 537, 551, 499, 507, 498, 543, 509, 504, 495, 529, 487, 477, 510, 495, 517, 526, 524, 542, 528, 498, 471, 492, 550, 491, 493, 523, 498, 481, 532, 528, 514, 532, 505, 535, 527, 538, 555, 474, 540, 520, 535, 519, 530, 481, 457, 530, 492, 521, 502, 519, 497, 522, 508, 493, 468, 510, 490, 515, 534] payload_rows_per_dst=[10884, 10888, 10880, 10950, 10964, 10976, 11054, 11094] dup_rows_per_dst=[5504, 5466, 5348, 5268, 5498, 5294, 5676, 5328] combine_rows_per_src=[10884, 10888, 10880, 10950, 10964, 10976, 11054, 11094] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625442816 dispatch_meta_bytes=458824 combine_payload_bytes=625442816 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=625901640 ingress_bytes=776437760 link_bytes=776437760 token_bytes=1251344456 weight_movement_bytes=150994944 total_nvlink_bytes=1402339400 weight_to_token_ratio=0.120666 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.846323716e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87824 payload_rows_remote=76940 dup_rows=43248 dup_rows_remote=37894 dedup_ratio=0.329956 router_tokens_per_expert=[516, 474, 513, 479, 525, 515, 520, 504, 475, 533, 512, 521, 500, 497, 540, 499, 538, 563, 498, 496, 495, 537, 507, 544, 505, 477, 496, 513, 528, 530, 511, 469, 494, 498, 514, 494, 501, 527, 494, 481, 536, 517, 552, 506, 512, 487, 521, 476, 504, 505, 522, 517, 523, 504, 520, 506, 540, 491, 482, 524, 536, 510, 504, 558, 507, 514, 527, 516, 478, 520, 526, 493, 548, 559, 521, 528, 478, 542, 522, 538, 536, 462, 533, 527, 511, 501, 510, 479, 527, 554, 497, 473, 477, 523, 494, 515, 512, 523, 514, 542, 488, 550, 532, 526, 494, 517, 490, 512, 517, 526, 506, 506, 494, 521, 501, 543, 521, 569, 487, 524, 503, 498, 479, 516, 505, 533, 477, 488] payload_rows_per_dst=[10940, 11058, 10848, 11040, 11074, 10884, 11044, 10936] dup_rows_per_dst=[5306, 5356, 5372, 5452, 5560, 5354, 5466, 5382] combine_rows_per_src=[10940, 11058, 10848, 11040, 11074, 10884, 11044, 10936] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=7 plans=1 dispatch_payload_bytes=624705536 dispatch_meta_bytes=458536 combine_payload_bytes=624705536 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625164072 ingress_bytes=700203008 link_bytes=700203008 token_bytes=1249869608 weight_movement_bytes=75497472 total_nvlink_bytes=1325367080 weight_to_token_ratio=0.060404 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.879559752e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87730 payload_rows_remote=76816 dup_rows=43342 dup_rows_remote=37894 dedup_ratio=0.330673 router_tokens_per_expert=[557, 489, 523, 523, 511, 511, 525, 492, 503, 510, 485, 514, 519, 523, 520, 480, 514, 537, 515, 545, 476, 495, 525, 525, 561, 545, 516, 520, 482, 528, 465, 496, 494, 545, 549, 555, 525, 526, 498, 512, 482, 493, 530, 505, 554, 490, 496, 530, 519, 508, 526, 526, 518, 504, 523, 481, 526, 478, 477, 509, 500, 457, 501, 516, 501, 518, 523, 531, 473, 542, 507, 512, 518, 501, 518, 480, 503, 499, 514, 541, 516, 543, 501, 487, 545, 546, 486, 498, 520, 509, 520, 528, 503, 490, 506, 509, 504, 515, 499, 513, 504, 525, 538, 498, 533, 496, 518, 528, 494, 496, 497, 529, 512, 513, 513, 523, 544, 459, 507, 506, 531, 546, 499, 506, 496, 500, 483, 540] payload_rows_per_dst=[10974, 11036, 11048, 10896, 10914, 10994, 10944, 10924] dup_rows_per_dst=[5396, 5454, 5520, 5242, 5448, 5420, 5430, 5432] combine_rows_per_src=[10974, 11036, 11048, 10896, 10914, 10994, 10944, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87258 payload_rows_remote=76352 dup_rows=43814 dup_rows_remote=38214 dedup_ratio=0.334274 router_tokens_per_expert=[525, 520, 512, 484, 515, 499, 476, 521, 541, 528, 553, 507, 486, 482, 510, 529, 526, 506, 517, 511, 476, 519, 489, 518, 538, 537, 519, 482, 474, 500, 511, 507, 482, 518, 512, 516, 530, 513, 524, 513, 479, 517, 486, 535, 515, 518, 542, 491, 481, 511, 507, 538, 550, 515, 529, 526, 470, 506, 526, 534, 513, 512, 505, 530, 547, 538, 537, 511, 506, 515, 512, 523, 527, 495, 523, 519, 503, 510, 515, 507, 519, 504, 492, 492, 500, 539, 517, 524, 534, 546, 527, 505, 504, 462, 519, 511, 553, 497, 518, 479, 523, 534, 555, 487, 492, 506, 497, 492, 488, 521, 478, 538, 523, 484, 449, 529, 518, 529, 529, 501, 480, 472, 556, 493, 509, 521, 538, 502] payload_rows_per_dst=[10912, 10850, 10972, 10906, 10976, 10960, 10914, 10768] dup_rows_per_dst=[5464, 5410, 5410, 5600, 5600, 5430, 5402, 5498] combine_rows_per_src=[10912, 10850, 10972, 10906, 10976, 10960, 10914, 10768] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87634 payload_rows_remote=76726 dup_rows=43438 dup_rows_remote=37932 dedup_ratio=0.331406 router_tokens_per_expert=[511, 493, 475, 529, 508, 514, 531, 520, 521, 525, 507, 521, 578, 532, 544, 463, 596, 498, 484, 527, 514, 481, 558, 498, 508, 531, 499, 509, 540, 529, 506, 551, 508, 474, 525, 493, 498, 508, 502, 545, 523, 464, 488, 538, 471, 512, 514, 511, 493, 510, 504, 512, 532, 517, 509, 490, 548, 529, 526, 518, 544, 492, 469, 502, 581, 505, 507, 486, 532, 515, 524, 534, 528, 485, 543, 520, 495, 494, 504, 501, 529, 502, 505, 522, 496, 530, 506, 481, 523, 526, 498, 484, 506, 484, 511, 490, 500, 527, 517, 499, 542, 512, 530, 485, 501, 538, 521, 508, 503, 528, 489, 507, 501, 512, 487, 515, 494, 523, 509, 501, 497, 500, 481, 534, 524, 500, 533, 501] payload_rows_per_dst=[10948, 10980, 10888, 10976, 11092, 10884, 10908, 10958] dup_rows_per_dst=[5596, 5678, 5260, 5414, 5416, 5302, 5506, 5266] combine_rows_per_src=[10948, 10980, 10888, 10976, 11092, 10884, 10908, 10958] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627474432 dispatch_meta_bytes=458600 combine_payload_bytes=627474432 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627933032 ingress_bytes=702971904 link_bytes=702971904 token_bytes=1255407464 weight_movement_bytes=75497472 total_nvlink_bytes=1330904936 weight_to_token_ratio=0.060138 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.914673250e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=630292480 dispatch_meta_bytes=459336 combine_payload_bytes=630292480 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=630751816 ingress_bytes=705789952 link_bytes=705789952 token_bytes=1261044296 weight_movement_bytes=75497472 total_nvlink_bytes=1336541768 weight_to_token_ratio=0.059869 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.950410066e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=629276672 dispatch_meta_bytes=458840 combine_payload_bytes=629276672 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629735512 ingress_bytes=780271616 link_bytes=780271616 token_bytes=1259012184 weight_movement_bytes=150994944 total_nvlink_bytes=1410007128 weight_to_token_ratio=0.119931 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.894942406e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=625475584 dispatch_meta_bytes=458264 combine_payload_bytes=625475584 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625933848 ingress_bytes=700973056 link_bytes=700973056 token_bytes=1251409432 weight_movement_bytes=75497472 total_nvlink_bytes=1326906904 weight_to_token_ratio=0.060330 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.889325044e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=628539392 dispatch_meta_bytes=458632 combine_payload_bytes=628539392 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=628998024 ingress_bytes=779534336 link_bytes=779534336 token_bytes=1257537416 weight_movement_bytes=150994944 total_nvlink_bytes=1408532360 weight_to_token_ratio=0.120072 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.885592658e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87406 payload_rows_remote=76442 dup_rows=43666 dup_rows_remote=38184 dedup_ratio=0.333145 router_tokens_per_expert=[461, 444, 505, 498, 513, 506, 549, 492, 525, 495, 497, 518, 517, 509, 503, 502, 533, 536, 519, 518, 502, 461, 474, 513, 539, 515, 509, 504, 502, 495, 515, 521, 515, 525, 525, 558, 518, 508, 532, 492, 517, 469, 482, 524, 511, 497, 531, 519, 513, 486, 487, 550, 487, 494, 522, 492, 501, 500, 534, 501, 516, 518, 523, 513, 502, 539, 501, 470, 516, 500, 524, 519, 526, 493, 528, 505, 537, 529, 527, 537, 487, 516, 505, 517, 503, 529, 501, 540, 545, 534, 480, 524, 519, 524, 486, 522, 535, 499, 494, 510, 503, 524, 525, 531, 514, 530, 462, 498, 508, 550, 529, 498, 507, 549, 548, 524, 512, 490, 514, 516, 515, 550, 501, 540, 507, 522, 488, 508] payload_rows_per_dst=[10808, 10908, 10964, 10940, 10918, 10978, 10898, 10992] dup_rows_per_dst=[5260, 5404, 5482, 5334, 5588, 5486, 5522, 5590] combine_rows_per_src=[10808, 10908, 10964, 10940, 10918, 10978, 10898, 10992] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87080 payload_rows_remote=76114 dup_rows=43992 dup_rows_remote=38600 dedup_ratio=0.335632 router_tokens_per_expert=[531, 535, 503, 515, 535, 537, 538, 515, 542, 495, 471, 505, 531, 516, 507, 539, 522, 492, 503, 512, 506, 505, 513, 510, 530, 532, 479, 526, 545, 493, 484, 527, 513, 486, 545, 518, 542, 526, 501, 552, 500, 494, 476, 486, 536, 528, 499, 502, 499, 537, 510, 496, 498, 535, 533, 512, 502, 523, 508, 494, 468, 528, 489, 517, 484, 489, 513, 522, 536, 534, 494, 556, 544, 477, 532, 506, 483, 517, 515, 527, 541, 461, 508, 525, 539, 509, 500, 496, 518, 532, 519, 496, 512, 493, 508, 494, 514, 521, 523, 494, 556, 507, 486, 488, 470, 522, 493, 545, 533, 499, 499, 546, 499, 501, 525, 519, 484, 512, 494, 491, 529, 491, 515, 533, 491, 498, 516, 515] payload_rows_per_dst=[10928, 10966, 10934, 10884, 10908, 10832, 10902, 10726] dup_rows_per_dst=[5702, 5392, 5474, 5414, 5550, 5470, 5490, 5500] combine_rows_per_src=[10928, 10966, 10934, 10884, 10908, 10832, 10902, 10726] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87814 payload_rows_remote=76636 dup_rows=43258 dup_rows_remote=37834 dedup_ratio=0.330032 router_tokens_per_expert=[536, 540, 545, 515, 466, 535, 484, 498, 524, 509, 510, 530, 521, 521, 522, 545, 526, 520, 444, 474, 494, 501, 543, 503, 553, 551, 504, 526, 489, 527, 530, 505, 518, 543, 502, 502, 508, 499, 501, 548, 524, 507, 489, 523, 548, 502, 495, 475, 502, 528, 499, 523, 513, 484, 513, 543, 507, 526, 510, 536, 506, 493, 484, 507, 518, 479, 521, 495, 530, 490, 501, 500, 544, 508, 505, 487, 567, 518, 518, 483, 530, 531, 533, 502, 542, 523, 493, 466, 519, 496, 486, 509, 534, 487, 518, 558, 461, 498, 521, 503, 509, 490, 501, 532, 542, 517, 522, 505, 507, 508, 514, 486, 551, 515, 556, 500, 529, 464, 488, 487, 520, 543, 519, 485, 516, 495, 535, 477] payload_rows_per_dst=[11178, 10970, 10906, 11080, 10852, 10990, 10960, 10878] dup_rows_per_dst=[5424, 5414, 5462, 5710, 5034, 5524, 5208, 5482] combine_rows_per_src=[11178, 10970, 10906, 11080, 10852, 10990, 10960, 10878] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626212864 dispatch_meta_bytes=458504 combine_payload_bytes=626212864 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626671368 ingress_bytes=701710336 link_bytes=701710336 token_bytes=1252884232 weight_movement_bytes=75497472 total_nvlink_bytes=1328381704 weight_to_token_ratio=0.060259 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.898674792e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87228 payload_rows_remote=76128 dup_rows=43844 dup_rows_remote=38296 dedup_ratio=0.334503 router_tokens_per_expert=[537, 562, 507, 518, 536, 517, 497, 537, 548, 537, 467, 530, 578, 520, 536, 535, 480, 511, 484, 500, 509, 478, 543, 516, 517, 504, 533, 531, 530, 512, 468, 525, 506, 528, 471, 526, 532, 512, 496, 508, 515, 512, 508, 474, 535, 494, 524, 471, 474, 499, 525, 503, 498, 483, 477, 507, 484, 476, 545, 532, 456, 526, 520, 497, 504, 541, 498, 532, 499, 481, 490, 487, 502, 538, 505, 563, 544, 526, 521, 503, 504, 501, 527, 558, 472, 517, 536, 517, 532, 512, 490, 508, 486, 487, 516, 480, 534, 510, 548, 526, 496, 516, 480, 529, 549, 536, 511, 538, 508, 484, 476, 496, 520, 527, 519, 553, 505, 535, 470, 475, 510, 510, 542, 485, 502, 513, 544, 495] payload_rows_per_dst=[10860, 10758, 10934, 11042, 10748, 10778, 11008, 11100] dup_rows_per_dst=[5422, 5118, 5660, 5604, 5336, 5372, 5784, 5548] combine_rows_per_src=[10860, 10758, 10934, 11042, 10748, 10778, 11008, 11100] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=623525888 dispatch_meta_bytes=458856 combine_payload_bytes=623525888 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=623984744 ingress_bytes=774520832 link_bytes=774520832 token_bytes=1247510632 weight_movement_bytes=150994944 total_nvlink_bytes=1398505576 weight_to_token_ratio=0.121037 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.040441 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.822014370e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87640 payload_rows_remote=76558 dup_rows=43432 dup_rows_remote=37778 dedup_ratio=0.331360 router_tokens_per_expert=[470, 490, 470, 574, 479, 537, 493, 500, 505, 506, 499, 562, 518, 499, 478, 512, 515, 514, 491, 520, 533, 552, 504, 511, 505, 532, 473, 508, 530, 536, 499, 499, 525, 513, 524, 517, 509, 517, 510, 519, 548, 485, 497, 520, 492, 509, 513, 536, 489, 509, 489, 513, 528, 519, 497, 502, 533, 502, 482, 505, 524, 533, 502, 554, 496, 486, 535, 501, 509, 498, 507, 494, 521, 526, 524, 519, 501, 489, 494, 548, 527, 527, 519, 550, 551, 515, 487, 534, 522, 515, 517, 520, 498, 532, 521, 533, 560, 490, 536, 529, 500, 510, 525, 507, 498, 509, 528, 491, 532, 503, 531, 470, 522, 498, 491, 528, 505, 475, 484, 520, 506, 503, 474, 463, 461, 514, 551, 577] payload_rows_per_dst=[10944, 10916, 10982, 10992, 10968, 11082, 10918, 10838] dup_rows_per_dst=[5240, 5528, 5486, 5370, 5328, 5654, 5520, 5306] combine_rows_per_src=[10944, 10916, 10982, 10992, 10968, 11082, 10918, 10838] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87326 payload_rows_remote=76376 dup_rows=43746 dup_rows_remote=38156 dedup_ratio=0.333755 router_tokens_per_expert=[550, 507, 551, 492, 533, 474, 475, 528, 500, 538, 530, 463, 484, 526, 520, 505, 541, 523, 510, 537, 475, 544, 527, 519, 481, 504, 494, 541, 503, 502, 518, 540, 489, 494, 495, 519, 507, 545, 522, 505, 512, 518, 505, 501, 496, 511, 502, 522, 511, 502, 474, 516, 539, 536, 505, 519, 531, 511, 540, 543, 501, 465, 517, 504, 537, 516, 520, 538, 493, 515, 510, 516, 490, 529, 492, 515, 524, 531, 548, 496, 478, 538, 528, 522, 470, 509, 538, 534, 543, 491, 505, 528, 503, 518, 510, 522, 541, 477, 518, 494, 478, 463, 503, 509, 483, 491, 465, 524, 552, 500, 510, 511, 519, 506, 533, 536, 522, 514, 520, 524, 492, 521, 523, 524, 487, 518, 489, 490] payload_rows_per_dst=[10938, 10884, 10924, 10990, 10950, 10946, 10730, 10964] dup_rows_per_dst=[5414, 5634, 5362, 5438, 5590, 5528, 5308, 5472] combine_rows_per_src=[10938, 10884, 10924, 10990, 10950, 10946, 10730, 10964] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=8 plans=1 dispatch_payload_bytes=627802112 dispatch_meta_bytes=457880 combine_payload_bytes=627802112 prefetch_bytes=0 reduce_bytes=75497472 egress_bytes=628259992 ingress_bytes=703299584 link_bytes=703299584 token_bytes=1256062104 weight_movement_bytes=75497472 total_nvlink_bytes=1331559576 weight_to_token_ratio=0.060106 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.918828694e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87586 payload_rows_remote=76628 dup_rows=43486 dup_rows_remote=38118 dedup_ratio=0.331772 router_tokens_per_expert=[488, 517, 485, 515, 520, 545, 526, 531, 516, 445, 523, 526, 505, 528, 512, 474, 518, 516, 518, 499, 527, 560, 528, 518, 519, 541, 501, 553, 501, 489, 525, 535, 523, 523, 542, 502, 499, 515, 476, 500, 513, 533, 502, 485, 519, 540, 530, 495, 500, 500, 524, 521, 544, 540, 480, 566, 498, 491, 512, 515, 512, 506, 472, 482, 507, 536, 520, 536, 514, 504, 518, 526, 511, 497, 488, 503, 498, 519, 506, 529, 517, 516, 544, 517, 512, 514, 487, 490, 499, 524, 526, 481, 536, 553, 537, 502, 482, 557, 488, 470, 534, 499, 503, 502, 507, 462, 473, 523, 574, 473, 516, 527, 487, 472, 473, 504, 500, 578, 483, 512, 528, 547, 527, 540, 468, 482, 528, 486] payload_rows_per_dst=[10830, 11014, 11020, 10958, 11046, 10932, 10942, 10844] dup_rows_per_dst=[5482, 5682, 5374, 5368, 5378, 5578, 5238, 5386] combine_rows_per_src=[10830, 11014, 11020, 10958, 11046, 10932, 10942, 10844] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87724 payload_rows_remote=76838 dup_rows=43348 dup_rows_remote=37920 dedup_ratio=0.330719 router_tokens_per_expert=[506, 520, 573, 503, 505, 497, 514, 524, 528, 481, 509, 510, 542, 538, 524, 502, 545, 544, 506, 533, 498, 518, 516, 507, 548, 522, 555, 487, 512, 481, 535, 491, 504, 514, 481, 518, 477, 488, 506, 512, 506, 555, 543, 524, 539, 536, 520, 490, 506, 522, 479, 510, 520, 563, 508, 502, 518, 481, 532, 487, 490, 499, 521, 478, 520, 503, 503, 542, 541, 500, 494, 526, 499, 541, 503, 480, 536, 477, 519, 526, 493, 502, 504, 496, 488, 500, 523, 485, 513, 507, 533, 520, 501, 510, 525, 536, 521, 486, 510, 504, 559, 536, 506, 502, 464, 513, 502, 505, 514, 518, 488, 529, 510, 522, 531, 557, 545, 472, 493, 502, 478, 487, 504, 515, 507, 525, 496, 486] payload_rows_per_dst=[11006, 11092, 10978, 10934, 10952, 10946, 10886, 10930] dup_rows_per_dst=[5546, 5504, 5448, 5298, 5468, 5326, 5428, 5330] combine_rows_per_src=[11006, 11092, 10978, 10934, 10952, 10946, 10886, 10930] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=623640576 dispatch_meta_bytes=457696 combine_payload_bytes=623640576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=624098272 ingress_bytes=699138048 link_bytes=699138048 token_bytes=1247738848 weight_movement_bytes=75497472 total_nvlink_bytes=1323236320 weight_to_token_ratio=0.060507 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.866054560e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627163136 dispatch_meta_bytes=457344 combine_payload_bytes=627163136 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627620480 ingress_bytes=853655552 link_bytes=853655552 token_bytes=1254783616 weight_movement_bytes=226492416 total_nvlink_bytes=1481276032 weight_to_token_ratio=0.180503 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082555401e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=625672192 dispatch_meta_bytes=458128 combine_payload_bytes=625672192 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626130320 ingress_bytes=852164608 link_bytes=852164608 token_bytes=1251802512 weight_movement_bytes=226492416 total_nvlink_bytes=1478294928 weight_to_token_ratio=0.180933 gemm_rows_defined=67072 gemm_rows_charged=69632 gemm_tail_pad_rows=2560 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.036765 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080664674e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=627736576 dispatch_meta_bytes=458984 combine_payload_bytes=627736576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628195560 ingress_bytes=703234048 link_bytes=703234048 token_bytes=1255932136 weight_movement_bytes=75497472 total_nvlink_bytes=1331429608 weight_to_token_ratio=0.060113 gemm_rows_defined=66304 gemm_rows_charged=69632 gemm_tail_pad_rows=3328 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.047794 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917997605e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=629456896 dispatch_meta_bytes=459032 combine_payload_bytes=629456896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629915928 ingress_bytes=704954368 link_bytes=704954368 token_bytes=1259372824 weight_movement_bytes=75497472 total_nvlink_bytes=1334870296 weight_to_token_ratio=0.059948 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.939813685e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87262 payload_rows_remote=76278 dup_rows=43810 dup_rows_remote=38266 dedup_ratio=0.334244 router_tokens_per_expert=[481, 544, 522, 495, 545, 500, 512, 489, 527, 490, 502, 529, 498, 494, 476, 507, 483, 520, 484, 471, 491, 531, 498, 499, 512, 535, 538, 504, 533, 548, 489, 512, 535, 531, 525, 506, 512, 510, 526, 527, 535, 541, 450, 532, 524, 518, 507, 485, 543, 505, 501, 529, 508, 501, 497, 495, 495, 505, 518, 507, 460, 485, 545, 491, 532, 493, 512, 503, 515, 504, 530, 538, 470, 487, 521, 519, 542, 498, 477, 567, 526, 503, 511, 532, 510, 504, 531, 524, 533, 488, 518, 517, 518, 493, 500, 551, 510, 531, 525, 533, 511, 523, 478, 519, 499, 521, 490, 515, 550, 536, 464, 522, 513, 531, 510, 485, 533, 495, 547, 547, 529, 483, 535, 485, 540, 493, 509, 499] payload_rows_per_dst=[10900, 10854, 10984, 10874, 10834, 10928, 10964, 10924] dup_rows_per_dst=[5322, 5442, 5544, 5296, 5582, 5590, 5490, 5544] combine_rows_per_src=[10900, 10854, 10984, 10874, 10834, 10928, 10964, 10924] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87338 payload_rows_remote=76424 dup_rows=43734 dup_rows_remote=38266 dedup_ratio=0.333664 router_tokens_per_expert=[512, 524, 495, 562, 493, 533, 542, 513, 493, 524, 524, 536, 504, 528, 496, 535, 482, 527, 489, 538, 515, 490, 522, 535, 533, 485, 538, 511, 525, 491, 506, 504, 498, 502, 511, 528, 533, 509, 554, 523, 487, 494, 466, 558, 504, 523, 475, 505, 532, 497, 501, 483, 503, 521, 492, 498, 489, 508, 515, 521, 505, 483, 495, 515, 513, 541, 528, 516, 524, 495, 534, 522, 500, 543, 520, 515, 491, 506, 527, 532, 466, 521, 525, 493, 471, 502, 533, 506, 506, 518, 512, 519, 565, 534, 540, 476, 442, 528, 535, 525, 495, 490, 533, 533, 547, 521, 535, 505, 502, 508, 479, 494, 502, 526, 473, 518, 523, 492, 509, 501, 509, 516, 523, 487, 483, 540, 518, 517] payload_rows_per_dst=[11050, 10914, 10874, 10736, 11000, 10940, 10944, 10880] dup_rows_per_dst=[5578, 5468, 5466, 5380, 5614, 5434, 5400, 5394] combine_rows_per_src=[11050, 10914, 10874, 10736, 11000, 10940, 10944, 10880] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87566 payload_rows_remote=76442 dup_rows=43506 dup_rows_remote=37938 dedup_ratio=0.331924 router_tokens_per_expert=[531, 476, 529, 510, 510, 521, 535, 509, 528, 565, 499, 498, 494, 510, 519, 513, 505, 490, 489, 497, 511, 539, 494, 543, 511, 490, 528, 515, 509, 494, 520, 501, 492, 534, 491, 534, 503, 508, 490, 522, 540, 508, 476, 511, 484, 457, 465, 510, 458, 525, 460, 509, 528, 519, 550, 517, 511, 552, 560, 468, 558, 489, 516, 533, 506, 523, 513, 491, 530, 509, 519, 545, 483, 521, 537, 537, 477, 519, 478, 484, 526, 536, 547, 497, 501, 514, 497, 494, 507, 537, 498, 501, 484, 485, 509, 548, 507, 495, 569, 502, 496, 515, 537, 533, 521, 482, 515, 514, 528, 526, 515, 494, 511, 515, 477, 531, 511, 540, 507, 505, 526, 524, 514, 527, 498, 510, 563, 514] payload_rows_per_dst=[11124, 10932, 10766, 10960, 10950, 10774, 11088, 10972] dup_rows_per_dst=[5568, 5340, 5284, 5546, 5394, 5390, 5410, 5574] combine_rows_per_src=[11124, 10932, 10766, 10960, 10950, 10774, 11088, 10972] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[1, 0, 0, 0, 0, 1, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=624869376 dispatch_meta_bytes=458176 combine_payload_bytes=624869376 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625327552 ingress_bytes=700366848 link_bytes=700366848 token_bytes=1250196928 weight_movement_bytes=75497472 total_nvlink_bytes=1325694400 weight_to_token_ratio=0.060388 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.881637474e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87610 payload_rows_remote=76556 dup_rows=43462 dup_rows_remote=37930 dedup_ratio=0.331589 router_tokens_per_expert=[492, 531, 538, 585, 494, 528, 524, 539, 508, 513, 496, 493, 489, 560, 532, 519, 481, 481, 526, 527, 504, 545, 559, 531, 507, 480, 497, 478, 509, 517, 488, 482, 483, 494, 495, 516, 532, 500, 521, 507, 516, 512, 524, 561, 497, 462, 527, 491, 493, 528, 498, 508, 480, 506, 533, 468, 501, 517, 547, 524, 513, 491, 540, 538, 522, 495, 506, 550, 516, 529, 514, 539, 525, 491, 485, 467, 524, 483, 521, 530, 486, 536, 531, 526, 523, 523, 498, 528, 537, 503, 515, 503, 520, 541, 510, 515, 542, 462, 530, 539, 538, 490, 508, 524, 484, 528, 525, 477, 497, 501, 530, 511, 489, 509, 533, 505, 515, 513, 488, 534, 510, 473, 494, 486, 503, 491, 533, 506] payload_rows_per_dst=[11106, 10780, 10872, 10892, 11016, 11138, 10752, 11054] dup_rows_per_dst=[5576, 5394, 5404, 5478, 5368, 5502, 5208, 5532] combine_rows_per_src=[11106, 10780, 10872, 10892, 11016, 11138, 10752, 11054] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87522 payload_rows_remote=76492 dup_rows=43550 dup_rows_remote=38068 dedup_ratio=0.332260 router_tokens_per_expert=[515, 495, 555, 517, 519, 519, 500, 470, 522, 556, 537, 537, 521, 487, 463, 499, 515, 529, 492, 516, 506, 521, 529, 504, 469, 502, 502, 503, 527, 540, 508, 555, 512, 495, 506, 462, 460, 499, 515, 520, 530, 453, 480, 464, 494, 521, 521, 491, 506, 479, 534, 515, 514, 541, 484, 497, 518, 506, 544, 506, 528, 535, 515, 533, 519, 531, 541, 520, 531, 506, 508, 494, 525, 522, 527, 531, 502, 497, 484, 540, 501, 526, 503, 515, 514, 492, 508, 519, 511, 544, 544, 540, 540, 505, 526, 468, 556, 540, 481, 483, 502, 566, 523, 530, 523, 534, 476, 543, 546, 504, 480, 467, 543, 517, 539, 471, 493, 495, 529, 482, 534, 510, 505, 489, 501, 544, 464, 524] payload_rows_per_dst=[10948, 10940, 10740, 11008, 11040, 11030, 10922, 10894] dup_rows_per_dst=[5476, 5496, 5106, 5502, 5516, 5482, 5586, 5386] combine_rows_per_src=[10948, 10940, 10740, 11008, 11040, 11030, 10922, 10894] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626065408 dispatch_meta_bytes=458760 combine_payload_bytes=626065408 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626524168 ingress_bytes=852557824 link_bytes=852557824 token_bytes=1252589576 weight_movement_bytes=226492416 total_nvlink_bytes=1479081992 weight_to_token_ratio=0.180819 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081163327e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=9 plans=1 dispatch_payload_bytes=626212864 dispatch_meta_bytes=457520 combine_payload_bytes=626212864 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=626670384 ingress_bytes=777207808 link_bytes=777207808 token_bytes=1252883248 weight_movement_bytes=150994944 total_nvlink_bytes=1403878192 weight_to_token_ratio=0.120518 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.856089008e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87436 payload_rows_remote=76436 dup_rows=43636 dup_rows_remote=38156 dedup_ratio=0.332916 router_tokens_per_expert=[493, 533, 516, 498, 499, 495, 501, 535, 483, 501, 504, 480, 486, 528, 516, 524, 540, 517, 538, 529, 491, 536, 470, 518, 537, 506, 485, 493, 523, 508, 524, 527, 502, 495, 483, 515, 511, 519, 492, 503, 518, 512, 538, 495, 525, 534, 511, 477, 514, 507, 562, 466, 490, 533, 483, 498, 514, 553, 500, 509, 496, 496, 517, 489, 521, 534, 504, 557, 486, 504, 507, 519, 487, 531, 531, 536, 525, 503, 509, 486, 518, 503, 516, 530, 531, 535, 531, 512, 496, 512, 489, 523, 525, 524, 539, 504, 484, 516, 489, 508, 511, 490, 522, 521, 520, 476, 545, 529, 548, 512, 533, 563, 463, 534, 511, 508, 501, 531, 517, 499, 530, 492, 523, 515, 526, 483, 506, 511] payload_rows_per_dst=[10808, 10994, 10812, 10934, 11000, 11062, 10912, 10914] dup_rows_per_dst=[5376, 5490, 5448, 5320, 5480, 5514, 5622, 5386] combine_rows_per_src=[10808, 10994, 10812, 10934, 11000, 11062, 10912, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87210 payload_rows_remote=76434 dup_rows=43862 dup_rows_remote=38432 dedup_ratio=0.334641 router_tokens_per_expert=[509, 539, 495, 474, 509, 501, 509, 472, 495, 531, 528, 496, 528, 490, 514, 485, 478, 459, 542, 519, 536, 590, 541, 499, 524, 521, 511, 514, 522, 544, 519, 535, 497, 516, 520, 511, 517, 514, 505, 478, 523, 528, 530, 525, 511, 510, 497, 509, 513, 493, 496, 475, 499, 499, 543, 502, 522, 517, 511, 502, 515, 504, 502, 510, 528, 537, 531, 517, 518, 500, 509, 549, 509, 524, 549, 503, 497, 488, 540, 505, 512, 488, 497, 493, 536, 495, 545, 512, 495, 485, 521, 552, 520, 518, 517, 525, 495, 489, 457, 570, 541, 541, 497, 502, 509, 492, 520, 527, 506, 484, 535, 532, 524, 506, 509, 461, 545, 511, 549, 502, 500, 488, 476, 496, 487, 514, 524, 509] payload_rows_per_dst=[10866, 11012, 10866, 10776, 11018, 10928, 10938, 10806] dup_rows_per_dst=[5284, 5696, 5516, 5430, 5590, 5494, 5456, 5396] combine_rows_per_src=[10866, 11012, 10866, 10776, 11018, 10928, 10938, 10806] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 2, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87500 payload_rows_remote=76400 dup_rows=43572 dup_rows_remote=38132 dedup_ratio=0.332428 router_tokens_per_expert=[507, 515, 502, 523, 526, 506, 502, 506, 482, 534, 559, 513, 498, 529, 498, 505, 491, 474, 513, 523, 488, 506, 477, 528, 513, 512, 505, 526, 508, 506, 533, 496, 490, 506, 501, 517, 531, 492, 517, 484, 530, 536, 537, 523, 548, 494, 493, 514, 551, 510, 529, 481, 505, 522, 535, 503, 513, 499, 487, 496, 492, 522, 495, 521, 481, 503, 458, 535, 549, 535, 475, 481, 516, 517, 527, 512, 510, 516, 537, 508, 524, 528, 556, 488, 495, 462, 512, 493, 494, 553, 513, 499, 519, 515, 519, 502, 539, 507, 500, 490, 516, 563, 481, 534, 489, 506, 523, 535, 502, 540, 533, 512, 525, 562, 522, 529, 482, 533, 491, 498, 515, 519, 536, 476, 509, 540, 456, 563] payload_rows_per_dst=[10998, 10846, 10890, 10934, 10806, 10956, 11100, 10970] dup_rows_per_dst=[5412, 5352, 5536, 5388, 5514, 5388, 5440, 5542] combine_rows_per_src=[10998, 10846, 10890, 10934, 10806, 10956, 11100, 10970] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 1, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=627146752 dispatch_meta_bytes=457944 combine_payload_bytes=627146752 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627604696 ingress_bytes=702644224 link_bytes=702644224 token_bytes=1254751448 weight_movement_bytes=75497472 total_nvlink_bytes=1330248920 weight_to_token_ratio=0.060169 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.910517807e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626622464 dispatch_meta_bytes=458240 combine_payload_bytes=626622464 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627080704 ingress_bytes=777617408 link_bytes=777617408 token_bytes=1253703168 weight_movement_bytes=150994944 total_nvlink_bytes=1404698112 weight_to_token_ratio=0.120439 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.861283313e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626163712 dispatch_meta_bytes=458368 combine_payload_bytes=626163712 prefetch_bytes=0 reduce_bytes=150994944 egress_bytes=626622080 ingress_bytes=777158656 link_bytes=777158656 token_bytes=1252785792 weight_movement_bytes=150994944 total_nvlink_bytes=1403780736 weight_to_token_ratio=0.120527 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.038603 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.855465692e-05 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=626147328 dispatch_meta_bytes=459464 combine_payload_bytes=626147328 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626606792 ingress_bytes=701644800 link_bytes=701644800 token_bytes=1252754120 weight_movement_bytes=75497472 total_nvlink_bytes=1328251592 weight_to_token_ratio=0.060265 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.897843704e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=625868800 dispatch_meta_bytes=458128 combine_payload_bytes=625868800 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=626326928 ingress_bytes=852361216 link_bytes=852361216 token_bytes=1252195728 weight_movement_bytes=226492416 total_nvlink_bytes=1478688144 weight_to_token_ratio=0.180876 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.080914001e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87520 payload_rows_remote=76538 dup_rows=43552 dup_rows_remote=38074 dedup_ratio=0.332275 router_tokens_per_expert=[502, 540, 510, 508, 571, 557, 541, 513, 497, 518, 447, 468, 525, 507, 493, 504, 553, 540, 530, 496, 506, 522, 503, 535, 496, 495, 544, 499, 502, 516, 557, 527, 532, 515, 499, 530, 552, 515, 505, 484, 515, 487, 537, 494, 469, 514, 549, 533, 517, 451, 477, 509, 477, 514, 524, 507, 541, 503, 520, 504, 512, 492, 522, 498, 476, 533, 489, 559, 491, 498, 481, 478, 515, 499, 559, 537, 520, 506, 537, 497, 493, 503, 531, 502, 528, 507, 505, 502, 503, 514, 519, 564, 521, 510, 509, 497, 532, 526, 497, 505, 530, 486, 504, 509, 494, 498, 505, 524, 510, 465, 483, 501, 508, 501, 526, 514, 488, 524, 523, 487, 524, 573, 492, 518, 491, 555, 517, 523] payload_rows_per_dst=[10882, 10968, 10982, 10854, 10960, 11004, 10800, 11070] dup_rows_per_dst=[5520, 5674, 5478, 5282, 5390, 5412, 5338, 5458] combine_rows_per_src=[10882, 10968, 10982, 10854, 10960, 11004, 10800, 11070] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87356 payload_rows_remote=76338 dup_rows=43716 dup_rows_remote=38264 dedup_ratio=0.333527 router_tokens_per_expert=[521, 466, 518, 526, 528, 530, 480, 503, 509, 528, 517, 528, 513, 486, 539, 485, 502, 541, 524, 504, 549, 490, 543, 503, 508, 523, 488, 508, 523, 545, 505, 479, 488, 471, 532, 496, 522, 512, 514, 488, 520, 493, 553, 557, 521, 501, 503, 526, 507, 512, 508, 501, 556, 529, 535, 525, 497, 518, 481, 534, 496, 570, 514, 496, 542, 488, 537, 525, 476, 530, 489, 511, 499, 479, 485, 520, 532, 454, 527, 498, 524, 522, 536, 511, 488, 525, 500, 494, 551, 529, 504, 488, 501, 530, 525, 507, 513, 546, 528, 519, 528, 523, 498, 530, 497, 556, 500, 478, 521, 483, 490, 509, 502, 483, 469, 523, 531, 536, 515, 529, 475, 517, 521, 489, 492, 535, 489, 496] payload_rows_per_dst=[10872, 11018, 10974, 10932, 10828, 10984, 10974, 10774] dup_rows_per_dst=[5482, 5452, 5420, 5626, 5356, 5486, 5464, 5430] combine_rows_per_src=[10872, 11018, 10974, 10932, 10828, 10984, 10974, 10774] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[1, 0, 0, 1, 0, 0, 0, 1] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87702 payload_rows_remote=76760 dup_rows=43370 dup_rows_remote=37940 dedup_ratio=0.330887 router_tokens_per_expert=[562, 521, 536, 478, 469, 492, 506, 505, 514, 506, 505, 528, 471, 523, 504, 506, 537, 517, 500, 521, 499, 556, 509, 519, 529, 531, 513, 507, 503, 502, 485, 515, 514, 482, 505, 513, 544, 497, 474, 541, 486, 538, 477, 507, 482, 526, 519, 514, 495, 491, 502, 509, 480, 511, 517, 489, 525, 497, 514, 527, 527, 525, 502, 477, 496, 511, 530, 522, 537, 539, 531, 515, 502, 517, 526, 521, 524, 496, 517, 537, 494, 533, 518, 476, 495, 560, 520, 546, 508, 470, 519, 551, 491, 489, 528, 533, 526, 470, 518, 549, 512, 513, 493, 525, 519, 509, 492, 501, 476, 534, 516, 530, 488, 532, 519, 519, 507, 504, 526, 543, 505, 517, 548, 497, 519, 467, 521, 513] payload_rows_per_dst=[10942, 10952, 11224, 11322, 10608, 10600, 11008, 11046] dup_rows_per_dst=[5430, 5442, 5790, 5802, 5086, 5086, 5330, 5404] combine_rows_per_src=[10942, 10952, 11224, 11322, 10608, 10600, 11008, 11046] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 2, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626999296 dispatch_meta_bytes=458448 combine_payload_bytes=626999296 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627457744 ingress_bytes=853491712 link_bytes=853491712 token_bytes=1254457040 weight_movement_bytes=226492416 total_nvlink_bytes=1480949456 weight_to_token_ratio=0.180550 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.045956 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082347629e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 2, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87054 payload_rows_remote=76038 dup_rows=44018 dup_rows_remote=38394 dedup_ratio=0.335831 router_tokens_per_expert=[520, 472, 503, 522, 552, 564, 495, 474, 494, 497, 493, 551, 530, 475, 515, 490, 489, 526, 492, 529, 485, 510, 498, 505, 506, 519, 463, 507, 510, 500, 534, 553, 550, 464, 543, 516, 493, 518, 521, 522, 489, 523, 494, 494, 510, 534, 523, 485, 538, 524, 533, 558, 474, 502, 503, 534, 515, 520, 503, 539, 494, 511, 493, 498, 544, 490, 505, 531, 516, 511, 476, 500, 498, 513, 542, 534, 519, 520, 523, 524, 503, 494, 522, 546, 516, 537, 507, 483, 530, 478, 520, 523, 482, 497, 517, 492, 519, 527, 516, 492, 495, 554, 551, 513, 498, 551, 491, 513, 501, 468, 499, 537, 527, 497, 522, 512, 491, 479, 483, 504, 622, 498, 510, 499, 470, 557, 519, 537] payload_rows_per_dst=[10598, 10812, 10734, 10952, 10964, 10896, 11082, 11016] dup_rows_per_dst=[5122, 5440, 5330, 5634, 5678, 5460, 5730, 5624] combine_rows_per_src=[10598, 10812, 10734, 10952, 10964, 10896, 11082, 11016] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625360896 dispatch_meta_bytes=458408 combine_payload_bytes=625360896 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625819304 ingress_bytes=700858368 link_bytes=700858368 token_bytes=1251180200 weight_movement_bytes=75497472 total_nvlink_bytes=1326677672 weight_to_token_ratio=0.060341 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.887870639e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87494 payload_rows_remote=76540 dup_rows=43578 dup_rows_remote=38152 dedup_ratio=0.332474 router_tokens_per_expert=[488, 516, 516, 482, 519, 521, 541, 503, 493, 535, 555, 503, 530, 518, 496, 520, 486, 526, 539, 499, 494, 512, 519, 485, 525, 497, 536, 487, 499, 496, 521, 528, 538, 522, 458, 517, 492, 539, 499, 515, 504, 570, 506, 528, 508, 511, 516, 533, 492, 492, 500, 520, 522, 530, 475, 516, 527, 471, 488, 515, 552, 530, 537, 515, 531, 551, 538, 468, 518, 529, 500, 506, 534, 471, 497, 495, 532, 495, 509, 500, 496, 518, 539, 506, 518, 483, 534, 554, 485, 550, 515, 511, 487, 513, 488, 493, 511, 504, 533, 517, 479, 506, 471, 500, 512, 521, 480, 551, 490, 499, 543, 507, 530, 536, 498, 520, 538, 506, 516, 559, 541, 503, 525, 505, 481, 509, 493, 465] payload_rows_per_dst=[11044, 10868, 10990, 10862, 10996, 10954, 10878, 10902] dup_rows_per_dst=[5428, 5430, 5522, 5502, 5352, 5426, 5370, 5548] combine_rows_per_src=[11044, 10868, 10990, 10862, 10996, 10954, 10878, 10902] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=10 plans=1 dispatch_payload_bytes=628817920 dispatch_meta_bytes=458800 combine_payload_bytes=628817920 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=629276720 ingress_bytes=704315392 link_bytes=704315392 token_bytes=1258094640 weight_movement_bytes=75497472 total_nvlink_bytes=1333592112 weight_to_token_ratio=0.060009 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.931710569e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87248 payload_rows_remote=76330 dup_rows=43824 dup_rows_remote=38266 dedup_ratio=0.334351 router_tokens_per_expert=[524, 512, 511, 470, 531, 524, 513, 493, 551, 509, 474, 508, 556, 491, 550, 493, 503, 495, 492, 515, 530, 531, 517, 501, 525, 493, 488, 515, 508, 527, 519, 514, 530, 519, 498, 474, 484, 503, 513, 547, 503, 529, 540, 511, 494, 508, 495, 499, 548, 512, 520, 508, 569, 516, 504, 501, 528, 506, 528, 491, 513, 492, 494, 508, 517, 522, 487, 546, 538, 548, 541, 506, 508, 505, 559, 545, 501, 483, 563, 520, 484, 533, 509, 519, 523, 463, 493, 534, 515, 483, 516, 509, 499, 529, 538, 506, 547, 490, 505, 461, 489, 506, 467, 487, 491, 522, 540, 508, 493, 498, 545, 498, 536, 498, 502, 499, 517, 491, 559, 502, 521, 527, 493, 507, 531, 534, 505, 457] payload_rows_per_dst=[10900, 10916, 10866, 10918, 11070, 10852, 10820, 10906] dup_rows_per_dst=[5520, 5430, 5428, 5558, 5708, 5454, 5274, 5452] combine_rows_per_src=[10900, 10916, 10866, 10918, 11070, 10852, 10820, 10906] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87456 payload_rows_remote=76522 dup_rows=43616 dup_rows_remote=38132 dedup_ratio=0.332764 router_tokens_per_expert=[489, 500, 510, 514, 538, 536, 541, 493, 501, 542, 530, 579, 482, 495, 521, 499, 508, 499, 518, 524, 479, 525, 546, 497, 535, 468, 531, 503, 531, 512, 501, 508, 536, 517, 519, 526, 516, 493, 520, 522, 500, 503, 500, 519, 552, 461, 505, 497, 452, 513, 497, 489, 502, 503, 531, 481, 505, 490, 507, 512, 489, 555, 500, 519, 540, 514, 500, 514, 490, 537, 505, 497, 504, 522, 546, 519, 485, 492, 501, 543, 474, 504, 524, 526, 502, 525, 487, 498, 500, 564, 533, 503, 522, 516, 525, 523, 507, 561, 533, 554, 506, 539, 501, 510, 518, 570, 508, 527, 488, 481, 507, 493, 520, 524, 508, 499, 478, 491, 519, 528, 499, 486, 501, 561, 489, 506, 531, 472] payload_rows_per_dst=[10978, 10788, 10832, 10890, 10934, 11046, 11038, 10950] dup_rows_per_dst=[5562, 5582, 5540, 5200, 5484, 5406, 5568, 5274] combine_rows_per_src=[10978, 10788, 10832, 10890, 10934, 11046, 11038, 10950] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87488 payload_rows_remote=76532 dup_rows=43584 dup_rows_remote=38120 dedup_ratio=0.332520 router_tokens_per_expert=[509, 502, 470, 547, 524, 529, 528, 551, 517, 517, 537, 525, 502, 498, 541, 521, 505, 525, 548, 453, 491, 514, 508, 494, 517, 492, 530, 513, 487, 530, 523, 506, 498, 531, 537, 520, 530, 519, 516, 494, 541, 495, 507, 507, 517, 529, 510, 510, 473, 512, 516, 523, 564, 523, 511, 473, 493, 546, 498, 499, 537, 468, 518, 542, 513, 475, 515, 486, 514, 488, 481, 502, 507, 482, 508, 504, 483, 506, 519, 524, 501, 521, 519, 519, 508, 500, 531, 503, 510, 525, 508, 516, 511, 539, 478, 533, 498, 510, 509, 508, 497, 499, 479, 503, 543, 535, 515, 527, 505, 518, 547, 517, 532, 484, 531, 537, 525, 504, 515, 504, 518, 468, 483, 503, 547, 513, 508, 514] payload_rows_per_dst=[11142, 10878, 10942, 10986, 10724, 10946, 10956, 10914] dup_rows_per_dst=[5494, 5394, 5580, 5406, 5290, 5498, 5464, 5458] combine_rows_per_src=[11142, 10878, 10942, 10986, 10724, 10946, 10956, 10914] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=622903296 dispatch_meta_bytes=457728 combine_payload_bytes=622903296 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=623361024 ingress_bytes=698400768 link_bytes=698400768 token_bytes=1246264320 weight_movement_bytes=75497472 total_nvlink_bytes=1321761792 weight_to_token_ratio=0.060579 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.856704812e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=627015680 dispatch_meta_bytes=458768 combine_payload_bytes=627015680 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627474448 ingress_bytes=778010624 link_bytes=778010624 token_bytes=1254490128 weight_movement_bytes=150994944 total_nvlink_bytes=1405485072 weight_to_token_ratio=0.120364 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.866269845e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[2, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626868224 dispatch_meta_bytes=458616 combine_payload_bytes=626868224 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627326840 ingress_bytes=702365696 link_bytes=702365696 token_bytes=1254195064 weight_movement_bytes=75497472 total_nvlink_bytes=1329692536 weight_to_token_ratio=0.060196 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.906985680e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=625295360 dispatch_meta_bytes=458384 combine_payload_bytes=625295360 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625753744 ingress_bytes=700792832 link_bytes=700792832 token_bytes=1251049104 weight_movement_bytes=75497472 total_nvlink_bytes=1326546576 weight_to_token_ratio=0.060347 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.887039550e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87498 payload_rows_remote=76552 dup_rows=43574 dup_rows_remote=38236 dedup_ratio=0.332443 router_tokens_per_expert=[514, 480, 525, 563, 485, 529, 525, 499, 519, 518, 534, 510, 512, 505, 529, 502, 520, 521, 518, 534, 524, 549, 535, 497, 506, 529, 496, 537, 483, 531, 509, 502, 504, 511, 515, 512, 540, 519, 505, 494, 542, 500, 494, 536, 481, 498, 467, 524, 492, 504, 503, 501, 532, 510, 486, 522, 490, 500, 507, 554, 531, 516, 529, 550, 489, 477, 511, 503, 519, 468, 524, 511, 481, 524, 546, 524, 521, 554, 536, 479, 526, 528, 490, 536, 527, 501, 529, 516, 511, 512, 535, 477, 509, 502, 494, 516, 488, 524, 465, 516, 490, 509, 500, 527, 531, 498, 522, 490, 526, 519, 516, 531, 496, 486, 504, 487, 475, 526, 488, 525, 525, 463, 537, 519, 515, 510, 529, 514] payload_rows_per_dst=[10888, 11020, 10946, 10994, 10916, 10958, 10902, 10874] dup_rows_per_dst=[5610, 5562, 5338, 5460, 5418, 5460, 5402, 5324] combine_rows_per_src=[10888, 11020, 10946, 10994, 10916, 10958, 10902, 10874] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626950144 dispatch_meta_bytes=458608 combine_payload_bytes=626950144 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627408752 ingress_bytes=702447616 link_bytes=702447616 token_bytes=1254358896 weight_movement_bytes=75497472 total_nvlink_bytes=1329856368 weight_to_token_ratio=0.060188 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.908024541e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87482 payload_rows_remote=76550 dup_rows=43590 dup_rows_remote=38100 dedup_ratio=0.332565 router_tokens_per_expert=[492, 464, 499, 507, 497, 510, 484, 518, 552, 515, 495, 544, 545, 527, 526, 553, 509, 487, 526, 540, 549, 496, 489, 474, 510, 539, 506, 515, 491, 517, 523, 540, 514, 497, 507, 530, 555, 493, 519, 508, 533, 539, 515, 521, 528, 525, 523, 511, 507, 501, 518, 495, 516, 503, 529, 557, 524, 526, 464, 469, 491, 518, 582, 479, 493, 526, 550, 491, 504, 502, 525, 552, 511, 526, 495, 537, 513, 514, 518, 530, 507, 474, 491, 528, 526, 468, 521, 481, 527, 492, 484, 534, 513, 504, 497, 513, 480, 470, 524, 495, 486, 519, 491, 504, 481, 533, 521, 492, 517, 545, 512, 522, 514, 508, 506, 494, 499, 542, 479, 489, 544, 529, 529, 507, 506, 527, 484, 504] payload_rows_per_dst=[10980, 10932, 11098, 10970, 11056, 10786, 10714, 10946] dup_rows_per_dst=[5476, 5490, 5538, 5388, 5518, 5334, 5470, 5376] combine_rows_per_src=[10980, 10932, 11098, 10970, 11056, 10786, 10714, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 1, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87392 payload_rows_remote=76454 dup_rows=43680 dup_rows_remote=38090 dedup_ratio=0.333252 router_tokens_per_expert=[483, 514, 552, 517, 516, 503, 540, 482, 508, 500, 525, 510, 533, 526, 512, 530, 504, 498, 504, 479, 521, 492, 509, 537, 526, 519, 539, 511, 488, 523, 492, 518, 499, 519, 480, 515, 540, 511, 555, 566, 525, 499, 493, 485, 525, 505, 527, 496, 481, 495, 530, 475, 549, 503, 542, 548, 541, 508, 534, 487, 510, 537, 472, 523, 522, 537, 511, 487, 525, 489, 504, 503, 515, 532, 479, 507, 507, 508, 518, 525, 551, 518, 518, 529, 531, 464, 494, 529, 488, 474, 505, 504, 504, 532, 511, 523, 481, 501, 520, 494, 518, 529, 480, 504, 531, 471, 535, 515, 520, 533, 503, 523, 527, 480, 491, 486, 507, 496, 550, 494, 496, 530, 487, 512, 494, 541, 547, 510] payload_rows_per_dst=[10938, 10964, 11008, 10962, 10838, 10838, 10858, 10986] dup_rows_per_dst=[5590, 5458, 5446, 5508, 5500, 5410, 5458, 5310] combine_rows_per_src=[10938, 10964, 11008, 10962, 10838, 10838, 10858, 10986] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=2 ep_rank=2 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627113984 dispatch_meta_bytes=459152 combine_payload_bytes=627113984 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=627573136 ingress_bytes=778108928 link_bytes=778108928 token_bytes=1254687120 weight_movement_bytes=150994944 total_nvlink_bytes=1405682064 weight_to_token_ratio=0.120345 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.867516478e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 1, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87494 payload_rows_remote=76628 dup_rows=43578 dup_rows_remote=38074 dedup_ratio=0.332474 router_tokens_per_expert=[521, 471, 534, 488, 486, 504, 496, 522, 508, 522, 486, 527, 541, 483, 519, 490, 517, 515, 496, 511, 507, 517, 540, 488, 502, 515, 532, 541, 480, 518, 476, 516, 524, 542, 505, 473, 544, 475, 554, 474, 542, 525, 492, 495, 475, 516, 500, 499, 484, 508, 514, 510, 529, 557, 544, 543, 460, 502, 541, 518, 535, 468, 489, 540, 452, 530, 513, 517, 532, 520, 501, 548, 549, 470, 494, 506, 519, 504, 504, 526, 505, 487, 492, 524, 513, 478, 497, 524, 530, 532, 524, 521, 538, 488, 512, 520, 500, 547, 502, 524, 521, 499, 561, 478, 496, 525, 525, 490, 459, 517, 548, 551, 501, 492, 528, 556, 513, 515, 523, 537, 495, 531, 544, 494, 496, 503, 555, 494] payload_rows_per_dst=[10846, 11020, 10862, 11004, 10876, 10866, 11002, 11018] dup_rows_per_dst=[5350, 5322, 5408, 5480, 5494, 5504, 5484, 5536] combine_rows_per_src=[10846, 11020, 10862, 11004, 10876, 10866, 11002, 11018] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87436 payload_rows_remote=76490 dup_rows=43636 dup_rows_remote=38222 dedup_ratio=0.332916 router_tokens_per_expert=[484, 469, 519, 521, 494, 509, 495, 481, 499, 502, 520, 516, 513, 520, 523, 534, 500, 526, 521, 543, 487, 537, 502, 473, 543, 488, 542, 508, 513, 513, 527, 502, 461, 531, 472, 543, 545, 494, 549, 523, 560, 510, 483, 478, 534, 560, 480, 528, 517, 496, 511, 513, 505, 549, 498, 498, 512, 512, 508, 473, 507, 524, 500, 508, 509, 495, 478, 483, 543, 479, 512, 490, 538, 531, 498, 517, 561, 559, 510, 512, 491, 492, 508, 535, 528, 525, 502, 504, 539, 527, 505, 461, 539, 513, 524, 513, 549, 523, 533, 507, 508, 517, 489, 497, 492, 504, 518, 533, 503, 521, 537, 498, 513, 526, 520, 519, 547, 534, 486, 499, 502, 466, 515, 511, 510, 479, 549, 504] payload_rows_per_dst=[10818, 11028, 10886, 10752, 10920, 11012, 11074, 10946] dup_rows_per_dst=[5380, 5422, 5560, 5182, 5510, 5656, 5512, 5414] combine_rows_per_src=[10818, 11028, 10886, 10752, 10920, 11012, 11074, 10946] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=1 ep_rank=1 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627097600 dispatch_meta_bytes=458600 combine_payload_bytes=627097600 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=627556200 ingress_bytes=702595072 link_bytes=702595072 token_bytes=1254653800 weight_movement_bytes=75497472 total_nvlink_bytes=1330151272 weight_to_token_ratio=0.060174 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.909894490e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=11 plans=1 dispatch_payload_bytes=626311168 dispatch_meta_bytes=458176 combine_payload_bytes=626311168 prefetch_bytes=75497472 reduce_bytes=226492416 egress_bytes=626769344 ingress_bytes=928301056 link_bytes=928301056 token_bytes=1253080512 weight_movement_bytes=301989888 total_nvlink_bytes=1555070400 weight_to_token_ratio=0.240998 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.044118 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.177216407e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87836 payload_rows_remote=76812 dup_rows=43236 dup_rows_remote=37848 dedup_ratio=0.329865 router_tokens_per_expert=[489, 461, 530, 517, 506, 481, 491, 535, 516, 474, 528, 512, 508, 533, 519, 522, 465, 515, 520, 512, 525, 498, 540, 532, 504, 491, 507, 533, 535, 512, 473, 527, 486, 480, 530, 516, 530, 482, 514, 479, 510, 498, 555, 526, 491, 515, 502, 514, 532, 527, 512, 494, 501, 528, 521, 511, 485, 496, 505, 541, 514, 532, 526, 525, 520, 476, 537, 525, 530, 484, 499, 493, 535, 509, 508, 498, 502, 525, 529, 536, 507, 487, 525, 487, 507, 481, 511, 519, 526, 527, 554, 495, 539, 505, 500, 522, 524, 525, 514, 476, 480, 535, 505, 504, 479, 520, 564, 555, 487, 492, 517, 510, 506, 530, 507, 532, 520, 522, 487, 492, 481, 533, 533, 539, 534, 528, 515, 503] payload_rows_per_dst=[10860, 11056, 10926, 11036, 11024, 11036, 10976, 10922] dup_rows_per_dst=[5384, 5322, 5330, 5464, 5388, 5348, 5398, 5602] combine_rows_per_src=[10860, 11056, 10926, 11036, 11024, 11036, 10976, 10922] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=0 unused_prefetch_slots=32 reduce_calls=1 reduce_slots=3 export_slots=3 remote_demand=0 remote_demand_peak=0 slot_utilization=0.000000 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 1, 1, 0, 1, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87438 payload_rows_remote=76560 dup_rows=43634 dup_rows_remote=38240 dedup_ratio=0.332901 router_tokens_per_expert=[497, 479, 500, 499, 523, 522, 532, 492, 527, 500, 561, 478, 541, 494, 534, 496, 501, 521, 479, 528, 523, 508, 521, 523, 514, 513, 520, 516, 513, 491, 489, 486, 535, 521, 533, 526, 527, 513, 509, 522, 511, 564, 496, 514, 513, 509, 543, 478, 524, 512, 530, 456, 516, 527, 538, 513, 490, 500, 506, 521, 483, 495, 511, 514, 501, 476, 469, 560, 498, 511, 516, 523, 495, 481, 536, 504, 523, 497, 543, 539, 512, 480, 546, 520, 496, 544, 506, 535, 536, 508, 520, 467, 487, 488, 462, 503, 495, 519, 525, 489, 505, 513, 544, 493, 512, 514, 538, 520, 510, 547, 473, 508, 528, 528, 525, 564, 492, 491, 495, 505, 509, 529, 514, 526, 531, 517, 522, 502] payload_rows_per_dst=[10916, 10804, 11074, 10878, 10952, 10912, 10928, 10974] dup_rows_per_dst=[5434, 5488, 5554, 5394, 5392, 5308, 5482, 5582] combine_rows_per_src=[10916, 10804, 11074, 10878, 10952, 10912, 10928, 10974] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=2 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15035201720 prefetch_bytes=830472192 reduce_bytes=981467136 weight_movement_bytes=1811939328 egress_bytes=7520352440 ingress_bytes=9326788608 link_bytes=9326788608 total_nvlink_bytes=16847141048 weight_to_token_ratio=0.120513 prefetch_share_of_ingress=0.089042 dedup_ratio=0.333220 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=13 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042126 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.856400666e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87242 payload_rows_remote=76376 dup_rows=43830 dup_rows_remote=38398 dedup_ratio=0.334396 router_tokens_per_expert=[550, 522, 506, 499, 510, 488, 473, 501, 497, 540, 479, 555, 522, 538, 476, 528, 487, 466, 518, 498, 495, 534, 533, 523, 510, 548, 535, 465, 491, 516, 497, 479, 482, 505, 519, 526, 493, 506, 530, 485, 510, 501, 497, 532, 547, 515, 506, 494, 543, 478, 511, 524, 552, 523, 530, 514, 493, 500, 506, 520, 515, 526, 493, 492, 514, 514, 514, 515, 526, 502, 534, 522, 520, 517, 515, 501, 516, 518, 505, 489, 559, 549, 508, 529, 521, 524, 502, 491, 551, 551, 520, 528, 512, 531, 491, 511, 501, 526, 483, 529, 497, 531, 511, 485, 524, 495, 506, 565, 500, 494, 508, 494, 453, 490, 500, 475, 493, 496, 518, 507, 560, 480, 521, 532, 554, 490, 526, 545] payload_rows_per_dst=[10858, 10886, 10826, 10942, 10898, 11114, 10866, 10852] dup_rows_per_dst=[5510, 5304, 5470, 5498, 5546, 5642, 5432, 5428] combine_rows_per_src=[10858, 10886, 10826, 10942, 10898, 11114, 10866, 10852] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=1 export_slots=1 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 0, 0, 0, 2] export_slots_per_peer=[1, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=5 ep_rank=5 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627736576 dispatch_meta_bytes=458808 combine_payload_bytes=627736576 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=628195384 ingress_bytes=703234048 link_bytes=703234048 token_bytes=1255931960 weight_movement_bytes=75497472 total_nvlink_bytes=1331429432 weight_to_token_ratio=0.060113 gemm_rows_defined=66432 gemm_rows_charged=69632 gemm_tail_pad_rows=3200 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.045956 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.917997605e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=1 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15037105384 prefetch_bytes=679477248 reduce_bytes=1132462080 weight_movement_bytes=1811939328 egress_bytes=7521305832 ingress_bytes=9327738880 link_bytes=9327738880 total_nvlink_bytes=16849044712 weight_to_token_ratio=0.120498 prefetch_share_of_ingress=0.072845 dedup_ratio=0.333215 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=15 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.041973 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.857404899e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=7 ep_rank=7 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=626606080 dispatch_meta_bytes=458848 combine_payload_bytes=626606080 prefetch_bytes=0 reduce_bytes=226492416 egress_bytes=627064928 ingress_bytes=853098496 link_bytes=853098496 token_bytes=1253671008 weight_movement_bytes=226492416 total_nvlink_bytes=1480163424 weight_to_token_ratio=0.180663 gemm_rows_defined=66560 gemm_rows_charged=69632 gemm_tail_pad_rows=3072 gemm_inner_pad_upper_bound=2159 gemm_tail_pad_fraction=0.044118 gemm_active_groups=17 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.081848976e-04 prefetch_bytes_per_gemm_flop=0.000000000e+00 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=2 export_slots=2 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 2, 0, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 1, 1, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 2, 0, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=4 ep_rank=4 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=629243904 dispatch_meta_bytes=458640 combine_payload_bytes=629243904 prefetch_bytes=75497472 reduce_bytes=75497472 egress_bytes=629702544 ingress_bytes=780238848 link_bytes=780238848 token_bytes=1258946448 weight_movement_bytes=150994944 total_nvlink_bytes=1409941392 weight_to_token_ratio=0.119938 gemm_rows_defined=66944 gemm_rows_charged=69632 gemm_tail_pad_rows=2688 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.038603 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=9.894526862e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=3 ep_rank=3 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=627179520 dispatch_meta_bytes=459200 combine_payload_bytes=627179520 prefetch_bytes=75497472 reduce_bytes=150994944 egress_bytes=627638720 ingress_bytes=853671936 link_bytes=853671936 token_bytes=1254818240 weight_movement_bytes=226492416 total_nvlink_bytes=1481310656 weight_to_token_ratio=0.180498 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=1.082576178e-04 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=6 ep_rank=6 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625672192 dispatch_meta_bytes=459096 combine_payload_bytes=625672192 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=626131288 ingress_bytes=701169664 link_bytes=701169664 token_bytes=1251803480 weight_movement_bytes=75497472 total_nvlink_bytes=1327300952 weight_to_token_ratio=0.060311 gemm_rows_defined=66816 gemm_rows_charged=69632 gemm_tail_pad_rows=2816 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.040441 gemm_active_groups=18 gemm_peak_group_rows=4352 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.891818310e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=5 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15060598984 prefetch_bytes=905969664 reduce_bytes=377487360 weight_movement_bytes=1283457024 egress_bytes=7533052104 ingress_bytes=8811003904 link_bytes=8811003904 total_nvlink_bytes=16344056008 weight_to_token_ratio=0.085220 prefetch_share_of_ingress=0.102823 dedup_ratio=0.332296 prefetch_slots=24 unused_prefetch_slots=360 slot_utilization=0.062500 reduce_slots=5 remote_demand=12 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27432 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.311327661e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=7 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15029296616 prefetch_bytes=679477248 reduce_bytes=981467136 weight_movement_bytes=1660944384 egress_bytes=7517396456 ingress_bytes=9172844544 link_bytes=9172844544 total_nvlink_bytes=16690241000 weight_to_token_ratio=0.110514 prefetch_share_of_ingress=0.074075 dedup_ratio=0.332846 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=13 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043045 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.693715048e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][routing] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 E=128 B=16 K=8 R=8 NvS=69600 hidden_size=4096 assignments=65536 payload_rows=87408 payload_rows_remote=76328 dup_rows=43664 dup_rows_remote=37982 dedup_ratio=0.333130 router_tokens_per_expert=[537, 517, 490, 502, 509, 516, 499, 532, 499, 478, 510, 507, 520, 541, 504, 503, 496, 511, 476, 506, 488, 537, 487, 513, 513, 515, 531, 504, 491, 506, 532, 500, 487, 532, 530, 523, 519, 539, 482, 519, 496, 519, 505, 451, 497, 518, 483, 525, 510, 530, 501, 549, 517, 500, 533, 514, 538, 534, 511, 522, 517, 496, 496, 547, 518, 524, 526, 529, 543, 529, 487, 493, 542, 503, 492, 513, 525, 492, 502, 576, 472, 505, 474, 499, 524, 540, 521, 482, 482, 511, 496, 498, 550, 541, 498, 475, 511, 506, 479, 491, 511, 502, 501, 523, 548, 533, 553, 556, 511, 487, 460, 543, 500, 478, 533, 504, 510, 530, 518, 541, 527, 512, 539, 519, 519, 507, 505, 507] payload_rows_per_dst=[11080, 10920, 10902, 11086, 10814, 10804, 10906, 10896] dup_rows_per_dst=[5682, 5496, 5354, 5544, 5350, 5332, 5524, 5382] combine_rows_per_src=[11080, 10920, 10902, 11086, 10814, 10804, 10906, 10896] +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=4 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15062205304 prefetch_bytes=830472192 reduce_bytes=603979776 weight_movement_bytes=1434451968 egress_bytes=7533855608 ingress_bytes=8962801664 link_bytes=8962801664 total_nvlink_bytes=16496657272 weight_to_token_ratio=0.095235 prefetch_share_of_ingress=0.092658 dedup_ratio=0.332306 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=8 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043199 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.471745100e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][weights] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 prefetch_calls=2 prefetch_slots=2 unused_prefetch_slots=30 reduce_calls=1 reduce_slots=0 export_slots=0 remote_demand=1 remote_demand_peak=1 slot_utilization=0.062500 weight_bytes_per_expert=37748736 grad_bytes_per_expert=75497472 prefetch_slots_per_owner=[0, 0, 0, 0, 2, 0, 0, 0] export_slots_per_peer=[0, 0, 0, 0, 0, 0, 0, 0] anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=3 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15043266560 prefetch_bytes=830472192 reduce_bytes=301989888 weight_movement_bytes=1132462080 egress_bytes=7524386816 ingress_bytes=8651341824 link_bytes=8651341824 total_nvlink_bytes=16175728640 weight_to_token_ratio=0.075280 prefetch_share_of_ingress=0.095993 dedup_ratio=0.333207 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=4 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043811 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.142599334e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=6 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15060207016 prefetch_bytes=830472192 reduce_bytes=679477248 weight_movement_bytes=1509949440 egress_bytes=7532856744 ingress_bytes=9037299712 link_bytes=9037299712 total_nvlink_bytes=16570156456 weight_to_token_ratio=0.100261 prefetch_share_of_ingress=0.091894 dedup_ratio=0.332484 prefetch_slots=22 unused_prefetch_slots=362 slot_utilization=0.057292 reduce_slots=9 remote_demand=11 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.043352 gemm_inner_pad_upper_bound=27305 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.550473442e-05 prefetch_bytes_per_gemm_flop=8.776296977e-06 anomalies=none +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][bytes] global_rank=0 ep_rank=0 iteration=11 iters_covered=1 layer=12 plans=1 dispatch_payload_bytes=625278976 dispatch_meta_bytes=457240 combine_payload_bytes=625278976 prefetch_bytes=75497472 reduce_bytes=0 egress_bytes=625736216 ingress_bytes=700776448 link_bytes=700776448 token_bytes=1251015192 weight_movement_bytes=75497472 total_nvlink_bytes=1326512664 weight_to_token_ratio=0.060349 gemm_rows_defined=66688 gemm_rows_charged=69632 gemm_tail_pad_rows=2944 gemm_inner_pad_upper_bound=2286 gemm_tail_pad_fraction=0.042279 gemm_active_groups=18 gemm_peak_group_rows=4224 gemm_flops_fwd=2628519985152 gemm_flops_step=7885559955456 bytes_per_gemm_flop=8.886831778e-05 prefetch_bytes_per_gemm_flop=9.574142157e-06 +WARNING:megatron.training.moonep_traffic_utils:[MOONEP_TRAFFIC][iteration_summary] global_rank=0 iteration=11 iters_covered=1 layers=12 plans=12 dispatch_calls=24 combine_calls=24 prefetch_calls=24 reduce_calls=12 token_bytes=15041520824 prefetch_bytes=679477248 reduce_bytes=1207959552 weight_movement_bytes=1887436800 egress_bytes=7523509432 ingress_bytes=9405448192 link_bytes=9405448192 total_nvlink_bytes=16928957624 weight_to_token_ratio=0.125482 prefetch_share_of_ingress=0.072243 dedup_ratio=0.332530 prefetch_slots=18 unused_prefetch_slots=366 slot_utilization=0.046875 reduce_slots=16 remote_demand=9 gemm_rows_charged=835584 gemm_tail_pad_fraction=0.042892 gemm_inner_pad_upper_bound=27051 gemm_flops_step=94626719465472 bytes_per_gemm_flop=9.939526854e-05 prefetch_bytes_per_gemm_flop=7.180606618e-06 anomalies=none + [2026-08-29 23:36:26] iteration 12/ 12 | consumed samples: 96 | elapsed time per iteration (ms): 654.1 | throughput per GPU (TFLOP/s/GPU): 276.4 | learning rate: 1.000000E-05 | global batch size: 8 | lm loss: 9.173823E+00 | loss scale: 1.0 | grad norm: 5.231 | number of skipped iterations: 0 | number of nan iterations: 0 | +[after training is done] datetime: 2026-08-29 23:36:26 +[rank0]:[W829 23:36:27.888393850 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank2]:[W829 23:36:28.831624295 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank6]:[W829 23:36:28.869060854 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank4]:[W829 23:36:28.929687794 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank7]:[W829 23:36:29.306615894 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank1]:[W829 23:36:29.309483607 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank5]:[W829 23:36:29.493688841 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +[rank3]:[W829 23:36:29.537342496 ProcessGroupNCCL.cpp:1505] Warning: WARNING: destroy_process_group() was not called before program exit, which can leak resources. For more info, please see https://pytorch.org/docs/stable/distributed.html#shutdown (function operator()) +W0829 23:36:31.969000 216540 torch/distributed/elastic/agent/server/api.py:719] Received 15 death signal, shutting down workers +W0829 23:36:31.970000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216725 closing signal SIGTERM +W0829 23:36:31.970000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216726 closing signal SIGTERM +W0829 23:36:31.971000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216727 closing signal SIGTERM +W0829 23:36:31.972000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216728 closing signal SIGTERM +W0829 23:36:31.973000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216729 closing signal SIGTERM +W0829 23:36:31.974000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216730 closing signal SIGTERM +W0829 23:36:31.974000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216731 closing signal SIGTERM +W0829 23:36:31.975000 216540 torch/distributed/elastic/multiprocessing/api.py:900] Sending process 216732 closing signal SIGTERM +Traceback (most recent call last): + File "/usr/local/bin/torchrun", line 8, in + sys.exit(main()) + ^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/errors/__init__.py", line 355, in wrapper + return f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 892, in main + run(args) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/run.py", line 883, in run + elastic_launch( + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 139, in __call__ + return launch_agent(self._config, self._entrypoint, list(args)) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/launcher/api.py", line 261, in launch_agent + result = agent.run() + ^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/metrics/api.py", line 138, in wrapper + result = f(*args, **kwargs) + ^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 711, in run + result = self._invoke_run(role) + ^^^^^^^^^^^^^^^^^^^^^^ + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/agent/server/api.py", line 870, in _invoke_run + time.sleep(monitor_interval) + File "/usr/local/lib/python3.12/dist-packages/torch/distributed/elastic/multiprocessing/api.py", line 84, in _terminate_process_handler + raise SignalException(f"Process {os.getpid()} got signal: {sigval}", sigval=sigval) +torch.distributed.elastic.multiprocessing.api.SignalException: Process 216540 got signal: 15 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv new file mode 100644 index 0000000000000000000000000000000000000000..46566f5ddc1f8978815beeba92a2c0253a808999 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/mem_samples.csv @@ -0,0 +1,48 @@ +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 4 +1, 4 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 +0, 4 +1, 4 +2, 4 +3, 4 +4, 4 +5, 4 +6, 4 +7, 1 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 4 +6, 4 +7, 4 +0, 0 +1, 0 +2, 0 +3, 0 +4, 0 +5, 0 +6, 0 +7, 0 +0, 1 +1, 1 +2, 1 +3, 1 +4, 1 +5, 1 +6, 1 +7, 1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env new file mode 100644 index 0000000000000000000000000000000000000000..09c700994ffd063eb84b9c5a311da96c220fe451 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/run.env @@ -0,0 +1,73 @@ +backend=moonep +config=QWEN3_30B +family=tierB +repo=/workspace/Megatron-MoE-Echo +branch=moonep_backend_async_grad_reduce_token_routing_analysis +commit=22fd6865 +moonep_source_dir=/workspace/MoonEP +moonep_source_branch=megatron-ddp-symmetric-params +moonep_source_commit=d8bc666 +num_layers=48 +hidden=2048 +ffn_hidden=6144 +moe_ffn_hidden=768 +num_experts=128 +topk=8 +experts_per_rank=16 +expert_params=4718592 +heads=32 +kv_groups=4 +kv_channels=128 +qk_layernorm=1 +shared_ffn=0 +shared_expert_overlap=0 +moe_token_dispatcher=flex +mla=0 +dense_first_layers=0 +moe_layers=48 +rotary_base=1000000 +rotary_scaling= +qkv_bias=0 +router_balancing=aux_loss +aux_loss_coeff=1e-3 +router_pre_softmax=0 +router_fusion=0 +router_dtype=fp32 +device_initiated_grouped_gemm=1 +norm_epsilon=1e-6 +seq_length=4096 +micro_batch_size=1 +global_batch_size=8 +num_microbatches=1 +assignments_per_rank=32768 +mean_tokens_per_expert=2048 +slot_policy=matched +slots_per_rank=16 +capacity_mode=none +capacity=none +cuda_graph=0 +probe_mode=1 +async_grad_reduce=1 +branch_role=probe +memory_dump_mode=none +memory_snapshot_path= +cuda_graph_warmup_steps=3 +recompute=none +force_balance=0 +train_iters=12 +ep_size=8 +echo_dispatch_overlap=1 +echo_dispatch_recompute=0 +ultraep_quota_min_tokens= +ultraep_weight_sync_join= +profile_enable=0 +kernel_trace=0 +fp8_mode=mxfp8 +fp8_recipe=mxfp8 +training_seed=1234 +learning_rate=1e-4 +min_learning_rate=1.0e-5 +lr_warmup_iters=5 +state_dump_enable=0 +state_dump_dir= +moonep_assert_no_pending_reduce=1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log new file mode 100644 index 0000000000000000000000000000000000000000..977ae41c4e74a66cfde0779d33b30cf8ff60f914 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/train.log @@ -0,0 +1,14 @@ +[23:36:37] WARN: PROBE_MODE forces CUDA_GRAPH=0 (routing instrumentation is not capture-safe) +[23:36:46] WARN: moonep_backend_async_grad_reduce_token_routing_analysis lacks --moe-router-fusion; QWEN3_30B runs unfused (timing only, routing identical) +============================================================== + QWEN3_30B / moonep (slots/rank=16 policy=matched) + E=128 K=8 H'=768 m=16 + L=48 seq=4096 gbs=8 (1 microbatches) + cap_mode=none cap=none cg=0 recompute=none + memory_dump=none snapshots=/workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_30B__moonep__matched__seq4096__gbs8__capnone__async/rep1/memory_snapshots/snapshot.pickle + repo=/workspace/Megatron-MoE-Echo @ 22fd6865 +============================================================== +W0829 23:36:47.546000 219093 torch/distributed/run.py:766] +W0829 23:36:47.546000 219093 torch/distributed/run.py:766] ***************************************** +W0829 23:36:47.546000 219093 torch/distributed/run.py:766] Setting OMP_NUM_THREADS environment variable for each process to be 1 in default, to avoid your system being overloaded, please further tune the variable for optimal performance in your application as needed. +W0829 23:36:47.546000 219093 torch/distributed/run.py:766] ***************************************** diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv new file mode 100644 index 0000000000000000000000000000000000000000..4a5fced99f1734da44f1d70acba0c9ba977e4460 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_attempts.tsv @@ -0,0 +1,3 @@ +model backend purpose capacity status peak_mib zero_drop drop_rate_pct required_capacity run_dir +QWEN3_235B moonep natural none OK 120129 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 +QWEN3_235B moonep forced none OK 120081 1 0 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv new file mode 100644 index 0000000000000000000000000000000000000000..ba332603fc3af4f3e6d852b6a64cf5e7f329f6b2 --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/capacity_selection.tsv @@ -0,0 +1,2 @@ +model backend status selected_capacity required_capacity drop_rate_pct peak_mib run_dir +QWEN3_235B moonep SELECTED none 0 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 diff --git a/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv new file mode 100644 index 0000000000000000000000000000000000000000..d2b3e06ae855ed5553473eab15818a7c3f7b91ea --- /dev/null +++ b/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/results.tsv @@ -0,0 +1,3 @@ +config backend policy slots seq gbs cap recompute rep status iter_ms_mean iter_ms_min peak_mib run_dir +QWEN3_235B moonep matched 16 8192 8 none none 1 OK 370.33 368.00 120129 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__async/rep1 +QWEN3_235B moonep matched 16 8192 8 none none+forcebal 1 OK 399.19 354.00 120081 /workspace/tierb-fixed-moonep-full/moonep/tierbprobe/tierbprobe_gbs8/QWEN3_235B__moonep__matched__seq8192__gbs8__capnone__forcebal__async/rep1